diff --git a/dbms/src/Common/CurrentMetrics.cpp b/dbms/src/Common/CurrentMetrics.cpp index 036f4c4f544..bfefc6a5093 100644 --- a/dbms/src/Common/CurrentMetrics.cpp +++ b/dbms/src/Common/CurrentMetrics.cpp @@ -62,6 +62,13 @@ M(DT_SnapshotOfPlaceIndex) \ M(DT_SnapshotOfBitmapFilter) \ M(DT_SnapshotOfDisaggReadNodeRead) \ + M(NumKeyspace) \ + M(NumIStorage) \ + M(DT_NumStorageDeltaMerge) \ + M(DT_NumSegment) \ + M(DT_NumMemTable) \ + M(DT_BytesMemTable) \ + M(DT_BytesMemTableAllocated) \ M(IOLimiterPendingBgWriteReq) \ M(IOLimiterPendingFgWriteReq) \ M(IOLimiterPendingBgReadReq) \ diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index e02b282ed51..76ee077cd80 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -349,6 +349,13 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva F(type_fg_write_alloc_bytes, {"type", "fg_write_alloc_bytes"}), \ F(type_bg_write_req_bytes, {"type", "bg_write_req_bytes"}), \ F(type_bg_write_alloc_bytes, {"type", "bg_write_alloc_bytes"})) \ + M(tiflash_storage_io_limiter_curr, \ + "Current limit bytes per second of Storage I/O limiter", \ + Gauge, \ + F(type_fg_read_bytes, {"type", "fg_read_bytes"}), \ + F(type_bg_read_bytes, {"type", "bg_read_bytes"}), \ + F(type_fg_write_bytes, {"type", "fg_write_bytes"}), \ + F(type_bg_write_bytes, {"type", "bg_write_bytes"})) \ M(tiflash_storage_rough_set_filter_rate, \ "Bucketed histogram of rough set filter rate", \ Histogram, \ diff --git a/dbms/src/IO/BaseFile/RateLimiter.cpp b/dbms/src/IO/BaseFile/RateLimiter.cpp index a4d857561ca..8afc0c3c140 100644 --- a/dbms/src/IO/BaseFile/RateLimiter.cpp +++ b/dbms/src/IO/BaseFile/RateLimiter.cpp @@ -526,6 +526,7 @@ void IORateLimiter::updateReadLimiter(Int64 bg_bytes, Int64 fg_bytes) { bg_read_limiter->updateMaxBytesPerSec(bg_bytes); } + GET_METRIC(tiflash_storage_io_limiter_curr, type_bg_read_bytes).Set(bg_bytes); if (fg_bytes == 0) { @@ -539,6 +540,7 @@ void IORateLimiter::updateReadLimiter(Int64 bg_bytes, Int64 fg_bytes) { fg_read_limiter->updateMaxBytesPerSec(fg_bytes); } + GET_METRIC(tiflash_storage_io_limiter_curr, type_fg_read_bytes).Set(fg_bytes); } void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes) @@ -556,6 +558,7 @@ void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes) { bg_write_limiter->updateMaxBytesPerSec(bg_bytes); } + GET_METRIC(tiflash_storage_io_limiter_curr, type_bg_write_bytes).Set(bg_bytes); if (fg_bytes == 0) { @@ -569,6 +572,7 @@ void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes) { fg_write_limiter->updateMaxBytesPerSec(fg_bytes); } + GET_METRIC(tiflash_storage_io_limiter_curr, type_fg_write_bytes).Set(fg_bytes); } void IORateLimiter::setBackgroundThreadIds(std::vector thread_ids) diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 90d20a1f381..ae398dd78e8 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -309,6 +309,11 @@ void AsynchronousMetrics::update() { GET_METRIC(tiflash_storage_s3_gc_status, type_owner).Set(1.0); } + else + { + // If the current node is not the owner, we reset the metric to 0 + GET_METRIC(tiflash_storage_s3_gc_status, type_owner).Set(0.0); + } } #if USE_MIMALLOC diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index d15e5887519..97bff2e4abe 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -68,7 +68,7 @@ class InterpreterCreateQuery : public IInterpreter ASTPtr query_ptr; Context & context; - std::string_view log_suffix; + std::string log_suffix; /// Using while loading database. ThreadPool * thread_pool = nullptr; diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h index 089b32ddb85..bbffd412fe2 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h @@ -94,8 +94,9 @@ class ColumnFile UInt64 getId() const { return id; } virtual size_t getRows() const { return 0; } - virtual size_t getBytes() const { return 0; }; - virtual size_t getDeletes() const { return 0; }; + virtual size_t getBytes() const { return 0; } + virtual size_t getAllocateBytes() const { return 0; } + virtual size_t getDeletes() const { return 0; } virtual Type getType() const = 0; @@ -130,14 +131,19 @@ class ColumnFile virtual ColumnFileReaderPtr getReader( const DMContext & context, const IColumnFileDataProviderPtr & data_provider, - const ColumnDefinesPtr & col_defs) const - = 0; + const ColumnDefinesPtr & col_defs) const = 0; /// Note: Only ColumnFileInMemory can be appendable. Other ColumnFiles (i.e. ColumnFilePersisted) have /// been persisted in the disk and their data will be immutable. virtual bool isAppendable() const { return false; } virtual void disableAppend() {} - virtual bool append( + + struct AppendResult + { + bool success = false; // whether the append is successful + size_t new_alloc_bytes = 0; // the new allocated bytes after append + }; + virtual AppendResult append( const DMContext & /*dm_context*/, const Block & /*data*/, size_t /*offset*/, diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp index 1990e4aafff..1d9d99cc499 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp @@ -42,6 +42,7 @@ void ColumnFileInMemory::fillColumns(const ColumnDefines & col_defs, size_t col_ // Copy data from cache const auto & type = getDataType(cd.id); auto col_data = type->createColumn(); + col_data->reserve(rows); col_data->insertRangeFrom(*(cache->block.getByPosition(col_offset).column), 0, rows); // Cast if need auto col_converted = convertColumnByColumnDefineIfNeed(type, std::move(col_data), cd); @@ -64,7 +65,13 @@ ColumnFileReaderPtr ColumnFileInMemory::getReader( return std::make_shared(*this, col_defs); } -bool ColumnFileInMemory::append( +void ColumnFileInMemory::disableAppend() +{ + disable_append = true; + // TODO: Call shrinkToFit() to release the extra memory of the cache block. +} + +ColumnFile::AppendResult ColumnFileInMemory::append( const DMContext & context, const Block & data, size_t offset, @@ -72,28 +79,31 @@ bool ColumnFileInMemory::append( size_t data_bytes) { if (disable_append) - return false; + return AppendResult{false, 0}; std::scoped_lock lock(cache->mutex); if (!isSameSchema(cache->block, data)) - return false; + return AppendResult{false, 0}; // check whether this instance overflows if (cache->block.rows() >= context.delta_cache_limit_rows || cache->block.bytes() >= context.delta_cache_limit_bytes) - return false; + return AppendResult{false, 0}; + size_t new_alloc_block_bytes = 0; for (size_t i = 0; i < cache->block.columns(); ++i) { const auto & col = data.getByPosition(i).column; const auto & cache_col = *cache->block.getByPosition(i).column; auto * mutable_cache_col = const_cast(&cache_col); + size_t alloc_bytes = mutable_cache_col->allocatedBytes(); mutable_cache_col->insertRangeFrom(*col, offset, limit); + new_alloc_block_bytes += mutable_cache_col->allocatedBytes() - alloc_bytes; } rows += limit; bytes += data_bytes; - return true; + return AppendResult{true, new_alloc_block_bytes}; } Block ColumnFileInMemory::readDataForFlush() const diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h index ec427033578..befb14e9c67 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h @@ -65,7 +65,8 @@ class ColumnFileInMemory : public ColumnFile Type getType() const override { return Type::INMEMORY_FILE; } size_t getRows() const override { return rows; } - size_t getBytes() const override { return bytes; }; + size_t getBytes() const override { return bytes; } + size_t getAllocateBytes() const override { return cache->block.allocatedBytes(); } CachePtr getCache() { return cache; } @@ -80,9 +81,13 @@ class ColumnFileInMemory : public ColumnFile const ColumnDefinesPtr & col_defs) const override; bool isAppendable() const override { return !disable_append; } - void disableAppend() override { disable_append = true; } - bool append(const DMContext & dm_context, const Block & data, size_t offset, size_t limit, size_t data_bytes) - override; + void disableAppend() override; + AppendResult append( + const DMContext & dm_context, + const Block & data, + size_t offset, + size_t limit, + size_t data_bytes) override; Block readDataForFlush() const; diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp index b1d33d130e7..89072a66ab1 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp @@ -271,8 +271,15 @@ size_t DeltaValueSpace::getTotalCacheBytes() const return mem_table_set->getBytes() + persisted_file_set->getTotalCacheBytes(); } +size_t DeltaValueSpace::getTotalAllocatedBytes() const +{ + std::scoped_lock lock(mutex); + return mem_table_set->getAllocatedBytes(); +} + size_t DeltaValueSpace::getValidCacheRows() const { + // FIXME: Seems that this function is the same as getTotalCacheRows(). std::scoped_lock lock(mutex); return mem_table_set->getRows() + persisted_file_set->getValidCacheRows(); } diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h index 5dbe6df20b6..8facdbefbed 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h @@ -84,7 +84,7 @@ class DeltaValueSpace /// Note that it's safe to do multiple flush concurrently but only one of them can succeed, /// and other thread's work is just a waste of resource. - /// So we only allow one flush task running at any time to aviod waste resource. + /// So we only allow one flush task running at any time to avoid waste resource. std::atomic_bool is_flushing = false; std::atomic last_try_flush_rows = 0; @@ -219,6 +219,7 @@ class DeltaValueSpace size_t getTotalCacheRows() const; size_t getTotalCacheBytes() const; + size_t getTotalAllocatedBytes() const; size_t getValidCacheRows() const; bool isFlushing() const { return is_flushing; } diff --git a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp index 0d1f4a5f578..952e94b4081 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp +++ b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.cpp @@ -22,10 +22,78 @@ #include #include -namespace DB +namespace CurrentMetrics { -namespace DM +extern const Metric DT_NumMemTable; +extern const Metric DT_BytesMemTable; +extern const Metric DT_BytesMemTableAllocated; +} // namespace CurrentMetrics + +namespace DB::DM +{ + +/// Member functions of MemTableSet::Statistic /// + +MemTableSet::Statistic::Statistic() + : holder_bytes(CurrentMetrics::DT_BytesMemTable, 0) + , holder_allocated_bytes(CurrentMetrics::DT_BytesMemTableAllocated, 0) +{} + +void MemTableSet::Statistic::append( + size_t rows_added, + size_t bytes_added, + size_t allocated_bytes_added, + size_t deletes_added, + size_t files_added) +{ + column_files_count += files_added; + rows += rows_added; + bytes += bytes_added; + allocated_bytes += allocated_bytes_added; + deletes += deletes_added; + // update the current metrics + holder_bytes.changeTo(bytes.load()); + holder_allocated_bytes.changeTo(allocated_bytes.load()); +} + +void MemTableSet::Statistic::resetTo( + size_t new_column_files_count, + size_t new_rows, + size_t new_bytes, + size_t new_allocated_bytes, + size_t new_deletes) +{ + column_files_count = new_column_files_count; + rows = new_rows; + bytes = new_bytes; + allocated_bytes = new_allocated_bytes; + deletes = new_deletes; + // update the current metrics + holder_bytes.changeTo(bytes.load()); + holder_allocated_bytes.changeTo(allocated_bytes.load()); +} + +/// Member functions of MemTableSet /// + +MemTableSet::MemTableSet(const ColumnFiles & in_memory_files) + : holder_counter(CurrentMetrics::DT_NumMemTable, 1) + , column_files(in_memory_files) + , log(Logger::get()) { + size_t new_rows = 0; + size_t new_bytes = 0; + size_t new_alloc_bytes = 0; + size_t new_deletes = 0; + for (const auto & file : column_files) + { + new_rows += file->getRows(); + new_bytes += file->getBytes(); + new_alloc_bytes += file->getAllocateBytes(); + new_deletes += file->getDeletes(); + } + stat.resetTo(column_files.size(), new_rows, new_bytes, new_alloc_bytes, new_deletes); +} + void MemTableSet::appendColumnFileInner(const ColumnFilePtr & column_file) { if (!column_files.empty()) @@ -38,11 +106,12 @@ void MemTableSet::appendColumnFileInner(const ColumnFilePtr & column_file) } column_files.push_back(column_file); - column_files_count = column_files.size(); - - rows += column_file->getRows(); - bytes += column_file->getBytes(); - deletes += column_file->getDeletes(); + stat.append( + column_file->getRows(), + column_file->getBytes(), + column_file->getAllocateBytes(), + column_file->getDeletes(), + /*files_added=*/1); } std::pair MemTableSet::diffColumnFiles( @@ -182,31 +251,37 @@ void MemTableSet::appendColumnFile(const ColumnFilePtr & column_file) void MemTableSet::appendToCache(DMContext & context, const Block & block, size_t offset, size_t limit) { // If the `column_files` is not empty, and the last `column_file` is a `ColumnInMemoryFile`, we will merge the newly block into the last `column_file`. - // Otherwise, create a new `ColumnInMemoryFile` and write into it. - bool success = false; + ColumnFile::AppendResult append_res; size_t append_bytes = block.bytes(offset, limit); if (!column_files.empty()) { auto & last_column_file = column_files.back(); if (last_column_file->isAppendable()) - success = last_column_file->append(context, block, offset, limit, append_bytes); + append_res = last_column_file->append(context, block, offset, limit, append_bytes); } - if (!success) + if (!append_res.success) { - auto schema = getSharedBlockSchemas(context)->getOrCreate(block); + /// Otherwise, create a new `ColumnInMemoryFile` and write into it. + // Try to reuse the global shared schema block. + auto schema = getSharedBlockSchemas(context)->getOrCreate(block); // Create a new column file. auto new_column_file = std::make_shared(schema); // Must append the empty `new_column_file` to `column_files` before appending data to it, // because `appendColumnFileInner` will update stats related to `column_files` but we will update stats relate to `new_column_file` here. appendColumnFileInner(new_column_file); - success = new_column_file->append(context, block, offset, limit, append_bytes); - if (unlikely(!success)) + append_res = new_column_file->append(context, block, offset, limit, append_bytes); + if (unlikely(!append_res.success)) throw Exception("Write to MemTableSet failed", ErrorCodes::LOGICAL_ERROR); } - rows += limit; - bytes += append_bytes; + + stat.append( // + limit, + append_bytes, + append_res.new_alloc_bytes, + /*deletes_added*/ 0, + /*files_added*/ 0); } void MemTableSet::appendDeleteRange(const RowKeyRange & delete_range) @@ -244,9 +319,9 @@ ColumnFileSetSnapshotPtr MemTableSet::createSnapshot( column_files.back()->disableAppend(); auto snap = std::make_shared(data_provider); - snap->rows = rows; - snap->bytes = bytes; - snap->deletes = deletes; + snap->rows = stat.rows; + snap->bytes = stat.bytes; + snap->deletes = stat.deletes; snap->column_files.reserve(column_files.size()); size_t total_rows = 0; @@ -273,11 +348,11 @@ ColumnFileSetSnapshotPtr MemTableSet::createSnapshot( // This may indicate that you forget to acquire a lock -- there are modifications // while this function is still running... RUNTIME_CHECK( - total_rows == rows && total_deletes == deletes, + total_rows == stat.rows && total_deletes == stat.deletes, total_rows, - rows.load(), + stat.rows.load(), total_deletes, - deletes.load()); + stat.deletes.load()); return snap; } @@ -312,7 +387,7 @@ ColumnFileFlushTaskPtr MemTableSet::buildFlushTask( cur_rows_offset += column_file->getRows(); cur_deletes_offset += column_file->getDeletes(); } - if (unlikely(flush_task->getFlushRows() != rows || flush_task->getFlushDeletes() != deletes)) + if (unlikely(flush_task->getFlushRows() != stat.rows || flush_task->getFlushDeletes() != stat.deletes)) { LOG_ERROR( log, @@ -320,8 +395,8 @@ ColumnFileFlushTaskPtr MemTableSet::buildFlushTask( "Files: {}", flush_task->getFlushRows(), flush_task->getFlushDeletes(), - rows.load(), - deletes.load(), + stat.rows.load(), + stat.deletes.load(), columnFilesToString(column_files)); throw Exception("Rows and deletes check failed.", ErrorCodes::LOGICAL_ERROR); } @@ -345,6 +420,7 @@ void MemTableSet::removeColumnFilesInFlushTask(const ColumnFileFlushTask & flush size_t new_rows = 0; size_t new_bytes = 0; + size_t new_alloc_bytes = 0; size_t new_deletes = 0; for (size_t i = tasks.size(); i < column_files.size(); ++i) { @@ -352,15 +428,17 @@ void MemTableSet::removeColumnFilesInFlushTask(const ColumnFileFlushTask & flush new_column_files.emplace_back(column_file); new_rows += column_file->getRows(); new_bytes += column_file->getBytes(); + new_alloc_bytes += column_file->getAllocateBytes(); new_deletes += column_file->getDeletes(); } column_files.swap(new_column_files); - column_files_count = column_files.size(); - rows = new_rows; - bytes = new_bytes; - deletes = new_deletes; + stat.resetTo( // + column_files.size(), + new_rows, + new_bytes, + new_alloc_bytes, + new_deletes); } -} // namespace DM -} // namespace DB +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.h b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.h index 2b91f5855c9..bef950dd628 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.h +++ b/dbms/src/Storages/DeltaMerge/Delta/MemTableSet.h @@ -18,9 +18,7 @@ #include #include -namespace DB -{ -namespace DM +namespace DB::DM { class MemTableSet; using MemTableSetPtr = std::shared_ptr; @@ -34,19 +32,60 @@ class MemTableSet : public std::enable_shared_from_this , private boost::noncopyable { +private: + struct Statistic + { + // TODO: check the proper memory_order when use this atomic variable + std::atomic column_files_count = 0; + std::atomic rows = 0; + std::atomic bytes = 0; + std::atomic allocated_bytes = 0; + std::atomic deletes = 0; + + CurrentMetrics::Increment holder_bytes; + CurrentMetrics::Increment holder_allocated_bytes; + + Statistic(); + + String info() const + { + return fmt::format( + "MemTableSet: {} column files, {} rows, {} bytes, {} deletes", + column_files_count.load(), + rows.load(), + bytes.load(), + deletes.load()); + } + + void append( + size_t rows_added, + size_t bytes_added, + size_t allocated_bytes_added, + size_t deletes_added, + size_t files_added); + + void resetTo( + size_t new_column_files_count, + size_t new_rows, + size_t new_bytes, + size_t new_allocated_bytes, + size_t new_deletes); + }; + #ifndef DBMS_PUBLIC_GTEST private: #else public: #endif + // Keep track of the number of mem-table in memory. + CurrentMetrics::Increment holder_counter; + // Note that we must update `column_files_count` for outer thread-safe after `column_files` changed ColumnFiles column_files; - // TODO: check the proper memory_order when use this atomic variable - std::atomic column_files_count; - std::atomic rows = 0; - std::atomic bytes = 0; - std::atomic deletes = 0; + // In order to avoid data-race and make it lightweight for accessing the statistic + // of mem-table, we use atomic variables to track the state of this MemTableSet. + Statistic stat; LoggerPtr log; @@ -54,18 +93,7 @@ class MemTableSet void appendColumnFileInner(const ColumnFilePtr & column_file); public: - explicit MemTableSet(const ColumnFiles & in_memory_files = {}) - : column_files(in_memory_files) - , log(Logger::get()) - { - column_files_count = column_files.size(); - for (const auto & file : column_files) - { - rows += file->getRows(); - bytes += file->getBytes(); - deletes += file->getDeletes(); - } - } + explicit MemTableSet(const ColumnFiles & in_memory_files = {}); /** * Resets the logger by using the one from the segment. @@ -75,20 +103,13 @@ class MemTableSet void resetLogger(const LoggerPtr & segment_log) { log = segment_log; } /// Thread safe part start - String info() const - { - return fmt::format( - "MemTableSet: {} column files, {} rows, {} bytes, {} deletes", - column_files_count.load(), - rows.load(), - bytes.load(), - deletes.load()); - } - - size_t getColumnFileCount() const { return column_files_count.load(); } - size_t getRows() const { return rows.load(); } - size_t getBytes() const { return bytes.load(); } - size_t getDeletes() const { return deletes.load(); } + String info() const { return stat.info(); } + + size_t getColumnFileCount() const { return stat.column_files_count.load(); } + size_t getRows() const { return stat.rows.load(); } + size_t getBytes() const { return stat.bytes.load(); } + size_t getAllocatedBytes() const { return stat.allocated_bytes.load(); } + size_t getDeletes() const { return stat.deletes.load(); } /// Thread safe part end /** @@ -153,5 +174,4 @@ class MemTableSet void removeColumnFilesInFlushTask(const ColumnFileFlushTask & flush_task); }; -} // namespace DM -} // namespace DB +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 6ed37ecec48..72542836863 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -93,6 +93,7 @@ struct SegmentStats UInt64 delta_persisted_column_files = 0; UInt64 delta_persisted_delete_ranges = 0; UInt64 delta_cache_size = 0; + UInt64 delta_cache_alloc_size = 0; UInt64 delta_index_size = 0; UInt64 stable_page_id = 0; @@ -109,6 +110,7 @@ using SegmentsStats = std::vector; struct StoreStats { + UInt64 column_count = 0; UInt64 segment_count = 0; UInt64 total_rows = 0; @@ -120,6 +122,7 @@ struct StoreStats Float64 delta_placed_rate = 0; UInt64 delta_cache_size = 0; + UInt64 delta_cache_alloc_size = 0; Float64 delta_cache_rate = 0; Float64 delta_cache_wasted_rate = 0; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp index 411d09a9f22..0e7936330eb 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Statistics.cpp @@ -17,9 +17,7 @@ #include #include -namespace DB -{ -namespace DM +namespace DB::DM { StoreStats DeltaMergeStore::getStoreStats() @@ -31,11 +29,13 @@ StoreStats DeltaMergeStore::getStoreStats() Int64 total_placed_rows = 0; Int64 total_delta_cache_rows = 0; - Float64 total_delta_cache_size = 0; + UInt64 total_delta_cache_size = 0; + UInt64 total_delta_cache_alloc_size = 0; Int64 total_delta_valid_cache_rows = 0; { std::shared_lock lock(read_write_mutex); stat.segment_count = segments.size(); + stat.column_count = original_table_columns.size(); for (const auto & [handle, segment] : segments) { @@ -64,6 +64,7 @@ StoreStats DeltaMergeStore::getStoreStats() total_delta_cache_rows += delta->getTotalCacheRows(); total_delta_cache_size += delta->getTotalCacheBytes(); + total_delta_cache_alloc_size += delta->getTotalAllocatedBytes(); total_delta_valid_cache_rows += delta->getValidCacheRows(); } @@ -87,6 +88,7 @@ StoreStats DeltaMergeStore::getStoreStats() stat.delta_placed_rate = static_cast(total_placed_rows) / stat.total_delta_rows; stat.delta_cache_size = total_delta_cache_size; + stat.delta_cache_alloc_size = total_delta_cache_alloc_size; stat.delta_cache_rate = static_cast(total_delta_valid_cache_rows) / stat.total_delta_rows; stat.delta_cache_wasted_rate = static_cast(total_delta_cache_rows - total_delta_valid_cache_rows) / total_delta_valid_cache_rows; @@ -153,8 +155,6 @@ SegmentsStats DeltaMergeStore::getSegmentsStats() SegmentStats stat; const auto & delta = segment->getDelta(); - const auto & delta_memtable = delta->getMemTableSet(); - const auto & delta_persisted = delta->getPersistedFileSet(); const auto & stable = segment->getStable(); stat.segment_id = segment->segmentId(); @@ -164,16 +164,25 @@ SegmentsStats DeltaMergeStore::getSegmentsStats() stat.size = segment->getEstimatedBytes(); stat.delta_rate = static_cast(delta->getRows()) / stat.rows; - stat.delta_memtable_rows = delta_memtable->getRows(); - stat.delta_memtable_size = delta_memtable->getBytes(); - stat.delta_memtable_column_files = delta_memtable->getColumnFileCount(); - stat.delta_memtable_delete_ranges = delta_memtable->getDeletes(); - stat.delta_persisted_page_id = delta_persisted->getId(); - stat.delta_persisted_rows = delta_persisted->getRows(); - stat.delta_persisted_size = delta_persisted->getBytes(); - stat.delta_persisted_column_files = delta_persisted->getColumnFileCount(); - stat.delta_persisted_delete_ranges = delta_persisted->getDeletes(); - stat.delta_cache_size = delta->getTotalCacheBytes(); + { + // Keep a copy to the shared_ptr of MemTableSet + const auto delta_memtable = delta->getMemTableSet(); + stat.delta_memtable_rows = delta_memtable->getRows(); + stat.delta_memtable_size = delta_memtable->getBytes(); + stat.delta_memtable_column_files = delta_memtable->getColumnFileCount(); + stat.delta_memtable_delete_ranges = delta_memtable->getDeletes(); + stat.delta_cache_size = delta_memtable->getBytes(); // FIXME: this is the same as delta_memtable_size + stat.delta_cache_alloc_size = delta_memtable->getAllocatedBytes(); + } + { + // Keep a copy to the shared_ptr of PersistedFileSet + const auto delta_persisted = delta->getPersistedFileSet(); + stat.delta_persisted_page_id = delta_persisted->getId(); + stat.delta_persisted_rows = delta_persisted->getRows(); + stat.delta_persisted_size = delta_persisted->getBytes(); + stat.delta_persisted_column_files = delta_persisted->getColumnFileCount(); + stat.delta_persisted_delete_ranges = delta_persisted->getDeletes(); + } stat.delta_index_size = delta->getDeltaIndexBytes(); stat.stable_page_id = stable->getId(); @@ -193,5 +202,4 @@ SegmentsStats DeltaMergeStore::getSegmentsStats() } -} // namespace DM -} // namespace DB +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Filter/In.h b/dbms/src/Storages/DeltaMerge/Filter/In.h index 7426502b857..f04ffb240a8 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/In.h +++ b/dbms/src/Storages/DeltaMerge/Filter/In.h @@ -47,7 +47,21 @@ class In : public RSOperator ","); buf.append("]}"); return buf.toString(); - }; + } + + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + obj->set("col", attr.col_name); + Poco::JSON::Array arr; + for (const auto & v : values) + { + arr.add(applyVisitor(FieldVisitorToDebugString(), v)); + } + obj->set("value", arr); + return obj; + } RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) override { diff --git a/dbms/src/Storages/DeltaMerge/Filter/IsNull.h b/dbms/src/Storages/DeltaMerge/Filter/IsNull.h index 72672cba273..39087bccdc5 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/IsNull.h +++ b/dbms/src/Storages/DeltaMerge/Filter/IsNull.h @@ -34,6 +34,14 @@ class IsNull : public RSOperator String toDebugString() override { return fmt::format(R"({{"op":"{}","col":"{}"}})", name(), attr.col_name); } + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + obj->set("col", attr.col_name); + return obj; + } + RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) override { RSResults results(pack_count, RSResult::Some); diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownFilter.h b/dbms/src/Storages/DeltaMerge/Filter/PushDownFilter.h index 97cdcc6d570..88ab4cd05cd 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownFilter.h +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownFilter.h @@ -14,6 +14,12 @@ #pragma once +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#pragma GCC diagnostic pop + +#include #include #include @@ -29,14 +35,14 @@ class PushDownFilter : public std::enable_shared_from_this public: PushDownFilter( const RSOperatorPtr & rs_operator_, - const ExpressionActionsPtr & beofre_where_, + const ExpressionActionsPtr & before_where_, const ExpressionActionsPtr & project_after_where_, const ColumnDefinesPtr & filter_columns_, const String filter_column_name_, const ExpressionActionsPtr & extra_cast_, const ColumnDefinesPtr & columns_after_cast_) : rs_operator(rs_operator_) - , before_where(beofre_where_) + , before_where(before_where_) , project_after_where(project_after_where_) , filter_column_name(std::move(filter_column_name_)) , filter_columns(filter_columns_) @@ -48,6 +54,16 @@ class PushDownFilter : public std::enable_shared_from_this : rs_operator(rs_operator_) {} + Poco::JSON::Object::Ptr toJSONObject() const + { + Poco::JSON::Object::Ptr json = new Poco::JSON::Object(); + if (rs_operator) + { + json->set("rs_operator", rs_operator->toJSONObject()); + } + return json; + } + // Rough set operator RSOperatorPtr rs_operator; // Filter expression actions and the name of the tmp filter column @@ -55,7 +71,7 @@ class PushDownFilter : public std::enable_shared_from_this const ExpressionActionsPtr before_where; // The projection after the filter, used to remove the tmp filter column // Used to construct the ExpressionBlockInputStream - // Note: ususally we will remove the tmp filter column in the LateMaterializationBlockInputStream, this only used for unexpected cases + // Note: usually we will remove the tmp filter column in the LateMaterializationBlockInputStream, this only used for unexpected cases const ExpressionActionsPtr project_after_where; const String filter_column_name; // The columns needed by the filter expression diff --git a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h index 03cafdd17f5..00580c56352 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h +++ b/dbms/src/Storages/DeltaMerge/Filter/RSOperator.h @@ -14,6 +14,11 @@ #pragma once +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#pragma GCC diagnostic pop + #include #include #include @@ -45,6 +50,7 @@ class RSOperator : public std::enable_shared_from_this virtual String name() = 0; virtual String toDebugString() = 0; + virtual Poco::JSON::Object::Ptr toJSONObject() = 0; virtual RSResults roughCheck(size_t start_pack, size_t pack_count, const RSCheckParam & param) = 0; @@ -73,6 +79,14 @@ class ColCmpVal : public RSOperator attr.col_name, applyVisitor(FieldVisitorToDebugString(), value)); } + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + obj->set("col", attr.col_name); + obj->set("value", applyVisitor(FieldVisitorToDebugString(), value)); + return obj; + } }; @@ -109,6 +123,18 @@ class LogicalOp : public RSOperator buf.append("]}"); return buf.toString(); } + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + Poco::JSON::Array arr; + for (const auto & child : children) + { + arr.add(child->toJSONObject()); + } + obj->set("children", arr); + return obj; + } }; #define GET_RSINDEX_FROM_PARAM_NOT_FOUND_RETURN_DIRECTLY(param, attr, rsindex, res) \ diff --git a/dbms/src/Storages/DeltaMerge/Filter/Unsupported.h b/dbms/src/Storages/DeltaMerge/Filter/Unsupported.h index 3b82bc27575..1be53d270e4 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/Unsupported.h +++ b/dbms/src/Storages/DeltaMerge/Filter/Unsupported.h @@ -39,6 +39,14 @@ class Unsupported : public RSOperator return fmt::format(R"({{"op":"{}","reason":"{}","content":"{}"}})", name(), reason, content); } + Poco::JSON::Object::Ptr toJSONObject() override + { + Poco::JSON::Object::Ptr obj = new Poco::JSON::Object(); + obj->set("op", name()); + obj->set("reason", reason); + return obj; + } + RSResults roughCheck(size_t /*start_pack*/, size_t pack_count, const RSCheckParam & /*param*/) override { return RSResults(pack_count, Some); diff --git a/dbms/src/Storages/DeltaMerge/ScanContext.cpp b/dbms/src/Storages/DeltaMerge/ScanContext.cpp index 8dae573e5d8..c34d0ca3133 100644 --- a/dbms/src/Storages/DeltaMerge/ScanContext.cpp +++ b/dbms/src/Storages/DeltaMerge/ScanContext.cpp @@ -18,6 +18,7 @@ #include #pragma GCC diagnostic pop #include +#include #include #include @@ -155,6 +156,11 @@ String ScanContext::toJson() const }; json->set("region_num_of_instance", to_json_array(region_num_of_instance)); + if (pushdown_executor) + { + json->set("pushdown", pushdown_executor->toJSONObject()); + } + std::stringstream buf; json->stringify(buf); return buf.str(); diff --git a/dbms/src/Storages/DeltaMerge/ScanContext.h b/dbms/src/Storages/DeltaMerge/ScanContext.h index 78321dd8737..b2f7770c267 100644 --- a/dbms/src/Storages/DeltaMerge/ScanContext.h +++ b/dbms/src/Storages/DeltaMerge/ScanContext.h @@ -28,6 +28,8 @@ namespace DB::DM { +class PushDownFilter; +using PushDownFilterPtr = std::shared_ptr; /// ScanContext is used to record statistical information in table scan for current query. /// For each table scan(one executor id), there is only one ScanContext. /// ScanContext helps to collect the statistical information of the table scan to show in `EXPLAIN ANALYZE`. @@ -85,6 +87,7 @@ class ScanContext std::atomic build_bitmap_time_ns{0}; const String resource_group_name; + PushDownFilterPtr pushdown_executor; explicit ScanContext(const String & name = "") : resource_group_name(name) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 02c2ea7ee3d..b922f08ec18 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -109,6 +109,7 @@ extern const Metric DT_SnapshotOfDeltaMerge; extern const Metric DT_SnapshotOfPlaceIndex; extern const Metric DT_SnapshotOfSegmentIngest; extern const Metric DT_SnapshotOfBitmapFilter; +extern const Metric DT_NumSegment; } // namespace CurrentMetrics namespace DB @@ -255,7 +256,8 @@ Segment::Segment( // PageIdU64 next_segment_id_, const DeltaValueSpacePtr & delta_, const StableValueSpacePtr & stable_) - : epoch(epoch_) + : holder_counter(CurrentMetrics::DT_NumSegment) + , epoch(epoch_) , rowkey_range(rowkey_range_) , is_common_handle(rowkey_range.is_common_handle) , rowkey_column_size(rowkey_range.rowkey_column_size) diff --git a/dbms/src/Storages/DeltaMerge/Segment.h b/dbms/src/Storages/DeltaMerge/Segment.h index 265982c0bd9..169825ac021 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.h +++ b/dbms/src/Storages/DeltaMerge/Segment.h @@ -734,6 +734,10 @@ class Segment #else public: #endif + + // Keep track of the number of segments in memory. + CurrentMetrics::Increment holder_counter; + /// The version of this segment. After split / merge / mergeDelta / replaceData, epoch got increased by 1. const UInt64 epoch; diff --git a/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp index 83f1b115932..16e8d925ef8 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool/StoragePool.cpp @@ -867,7 +867,7 @@ void StoragePool::drop() } } -PageIdU64 StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, const char * who) const +PageIdU64 StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, [[maybe_unused]] const char * who) const { // In case that there is a DTFile created on disk but TiFlash crashes without persisting the ID. // After TiFlash process restored, the ID will be inserted into the stable delegator, but we may @@ -893,7 +893,7 @@ PageIdU64 StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, c // else there is a DTFile with that id, continue to acquire a new ID. LOG_WARNING( logger, - "The DTFile is already exists, continute to acquire another ID. call={} path={} file_id={}", + "The DTFile is already exists, continue to acquire another ID. call={} path={} file_id={}", who, existed_path, dtfile_id); diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index cd7ed03b5ea..f63c0d98488 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -14,6 +14,10 @@ #include +namespace CurrentMetrics +{ +extern const Metric NumIStorage; +} // namespace CurrentMetrics namespace DB { @@ -24,6 +28,14 @@ extern const int DEADLOCK_AVOIDED; extern const int TABLE_IS_DROPPED; } // namespace ErrorCodes +IStorage::IStorage() + : holder_counter(CurrentMetrics::NumIStorage, 1) +{} + +IStorage::IStorage(ColumnsDescription columns_) + : ITableDeclaration(std::move(columns_)) + , holder_counter(CurrentMetrics::NumIStorage, 1) +{} RWLock::LockHolder IStorage::tryLockTimed( const RWLockPtr & rwlock, diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index bc75af28843..c1954ee5b14 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -65,6 +65,10 @@ class IStorage , public ITableDeclaration { public: + IStorage(); + + explicit IStorage(ColumnsDescription columns_); + /// The main name of the table type (for example, StorageDeltaMerge). virtual std::string getName() const = 0; @@ -349,6 +353,8 @@ class IStorage /// DROP-like queries take this lock for write (lockExclusively), to be sure /// that all table threads finished. mutable RWLockPtr drop_lock = RWLock::create(); + + CurrentMetrics::Increment holder_counter; }; /// table name -> table diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index 8635da352f7..8296ada0da9 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -37,7 +37,7 @@ class ITableDeclaration Block getSampleBlockNonMaterialized() const; Block getSampleBlockForColumns(const Names & column_names) const; - /** The hidden coloumns will not be returned. Mainly for INSERT query. + /** The hidden columns will not be returned. Mainly for INSERT query. */ Block getSampleBlockNoHidden() const; Block getSampleBlockNonMaterializedNoHidden() const; diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index d7dcb69490b..cda4cc42598 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,10 @@ #include #include +namespace CurrentMetrics +{ +extern const Metric DT_NumStorageDeltaMerge; +} // namespace CurrentMetrics namespace DB { @@ -87,6 +92,7 @@ StorageDeltaMerge::StorageDeltaMerge( Context & global_context_) : IManageableStorage{columns_, tombstone} , data_path_contains_database_name(db_engine != "TiFlash") + , holder_counter(CurrentMetrics::DT_NumStorageDeltaMerge, 1) , store_inited(false) , max_column_id_used(0) , global_context(global_context_.getGlobalContext()) @@ -979,6 +985,7 @@ BlockInputStreams StorageDeltaMerge::read( auto runtime_filter_list = parseRuntimeFilterList(query_info, context); const auto & scan_context = mvcc_query_info.scan_context; + scan_context->pushdown_executor = filter; auto streams = store->read( context, @@ -1073,6 +1080,7 @@ void StorageDeltaMerge::read( auto runtime_filter_list = parseRuntimeFilterList(query_info, context); const auto & scan_context = mvcc_query_info.scan_context; + scan_context->pushdown_executor = filter; store->read( exec_context_, diff --git a/dbms/src/Storages/StorageDeltaMerge.h b/dbms/src/Storages/StorageDeltaMerge.h index 808f73629f1..c3668672f31 100644 --- a/dbms/src/Storages/StorageDeltaMerge.h +++ b/dbms/src/Storages/StorageDeltaMerge.h @@ -294,6 +294,9 @@ class StorageDeltaMerge }; const bool data_path_contains_database_name = false; + // Keep track of the number of StorageDeltaMerge in memory. + CurrentMetrics::Increment holder_counter; + mutable std::mutex store_mutex; std::unique_ptr table_column_info; // After create DeltaMergeStore object, it is deprecated. diff --git a/dbms/src/Storages/System/StorageSystemDTSegments.cpp b/dbms/src/Storages/System/StorageSystemDTSegments.cpp index c10165141c2..221ee6fa85d 100644 --- a/dbms/src/Storages/System/StorageSystemDTSegments.cpp +++ b/dbms/src/Storages/System/StorageSystemDTSegments.cpp @@ -57,6 +57,7 @@ StorageSystemDTSegments::StorageSystemDTSegments(const std::string & name_) {"delta_persisted_column_files", std::make_shared()}, {"delta_persisted_delete_ranges", std::make_shared()}, {"delta_cache_size", std::make_shared()}, + {"delta_cache_alloc_size", std::make_shared()}, {"delta_index_size", std::make_shared()}, {"stable_page_id", std::make_shared()}, @@ -140,6 +141,7 @@ BlockInputStreams StorageSystemDTSegments::read( res_columns[j++]->insert(stat.delta_persisted_column_files); res_columns[j++]->insert(stat.delta_persisted_delete_ranges); res_columns[j++]->insert(stat.delta_cache_size); + res_columns[j++]->insert(stat.delta_cache_alloc_size); res_columns[j++]->insert(stat.delta_index_size); res_columns[j++]->insert(stat.stable_page_id); diff --git a/dbms/src/Storages/System/StorageSystemDTTables.cpp b/dbms/src/Storages/System/StorageSystemDTTables.cpp index e4dbdbc2f64..df4647f1519 100644 --- a/dbms/src/Storages/System/StorageSystemDTTables.cpp +++ b/dbms/src/Storages/System/StorageSystemDTTables.cpp @@ -40,6 +40,7 @@ StorageSystemDTTables::StorageSystemDTTables(const std::string & name_) {"table_id", std::make_shared()}, {"is_tombstone", std::make_shared()}, + {"column_count", std::make_shared()}, {"segment_count", std::make_shared()}, {"total_rows", std::make_shared()}, @@ -51,6 +52,7 @@ StorageSystemDTTables::StorageSystemDTTables(const std::string & name_) {"delta_placed_rate", std::make_shared()}, {"delta_cache_size", std::make_shared()}, + {"delta_cache_alloc_size", std::make_shared()}, {"delta_cache_rate", std::make_shared()}, {"delta_cache_wasted_rate", std::make_shared()}, @@ -154,6 +156,7 @@ BlockInputStreams StorageSystemDTTables::read( res_columns[j++]->insert(table_id); res_columns[j++]->insert(dm_storage->getTombstone()); + res_columns[j++]->insert(stat.column_count); res_columns[j++]->insert(stat.segment_count); res_columns[j++]->insert(stat.total_rows); @@ -165,6 +168,7 @@ BlockInputStreams StorageSystemDTTables::read( res_columns[j++]->insert(stat.delta_placed_rate); res_columns[j++]->insert(stat.delta_cache_size); + res_columns[j++]->insert(stat.delta_cache_alloc_size); res_columns[j++]->insert(stat.delta_cache_rate); res_columns[j++]->insert(stat.delta_cache_wasted_rate); diff --git a/dbms/src/TestUtils/gtests_dbms_main.cpp b/dbms/src/TestUtils/gtests_dbms_main.cpp index edeb44866db..345ea6c3796 100644 --- a/dbms/src/TestUtils/gtests_dbms_main.cpp +++ b/dbms/src/TestUtils/gtests_dbms_main.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -66,7 +67,8 @@ int main(int argc, char ** argv) install_fault_signal_handlers({SIGSEGV, SIGILL, SIGFPE, SIGABRT, SIGTERM}); bool enable_colors = isatty(STDERR_FILENO) && isatty(STDOUT_FILENO); - DB::tests::TiFlashTestEnv::setupLogger("trace", std::cerr, enable_colors); + const auto log_level = Utils::normalizeLogLevel(Poco::Environment::get("LOG_LEVEL", "trace")); + DB::tests::TiFlashTestEnv::setupLogger(log_level, std::cerr, enable_colors); auto run_mode = DB::PageStorageRunMode::ONLY_V3; DB::tests::TiFlashTestEnv::initializeGlobalContext(/*testdata_path*/ {}, run_mode); DB::ServerInfo server_info; diff --git a/dbms/src/TiDB/Schema/SchemaSyncService.cpp b/dbms/src/TiDB/Schema/SchemaSyncService.cpp index 31dae16212c..99fec6cbe9f 100644 --- a/dbms/src/TiDB/Schema/SchemaSyncService.cpp +++ b/dbms/src/TiDB/Schema/SchemaSyncService.cpp @@ -29,6 +29,11 @@ #include +namespace CurrentMetrics +{ +extern const Metric NumKeyspace; +} // namespace CurrentMetrics + namespace DB { namespace ErrorCodes @@ -73,6 +78,7 @@ void SchemaSyncService::addKeyspaceGCTasks() std::unique_lock lock(keyspace_map_mutex); for (auto const iter : keyspaces) { + // Already exist auto keyspace = iter.first; if (keyspace_handle_map.contains(keyspace)) continue; @@ -126,6 +132,7 @@ void SchemaSyncService::addKeyspaceGCTasks() keyspace_handle_map.emplace(keyspace, task_handle); num_add_tasks += 1; + CurrentMetrics::add(CurrentMetrics::NumKeyspace, 1); } auto log_level = num_add_tasks > 0 ? Poco::Message::PRIO_INFORMATION : Poco::Message::PRIO_DEBUG; @@ -158,6 +165,7 @@ void SchemaSyncService::removeKeyspaceGCTasks() PDClientHelper::removeKeyspaceGCSafepoint(keyspace); keyspace_gc_context.erase(keyspace); // clear the last gc safepoint num_remove_tasks += 1; + CurrentMetrics::sub(CurrentMetrics::NumKeyspace, 1); } auto log_level = num_remove_tasks > 0 ? Poco::Message::PRIO_INFORMATION : Poco::Message::PRIO_DEBUG; diff --git a/libs/libcommon/include/common/logger_util.h b/libs/libcommon/include/common/logger_util.h new file mode 100644 index 00000000000..f311a447fb8 --- /dev/null +++ b/libs/libcommon/include/common/logger_util.h @@ -0,0 +1,36 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +namespace Utils +{ + +inline std::string normalizeLogLevel(const std::string & log_level) +{ + std::string norm = Poco::toLower(log_level); + // normalize + // info -> information + // warn -> warning + if (norm == "info") + return "information"; + else if (norm == "warn") + return "warning"; + else + return norm; +} + +} // namespace Utils diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index a3975226bf1..dbfa684de6e 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -78,6 +78,7 @@ #include #include #include +#include #include #include #include @@ -619,7 +620,8 @@ static void terminate_handler() log << "what(): " << e.what() << std::endl; } catch (...) - {} + { + } log << "Stack trace:\n\n" << StackTrace().toString() << std::endl; } @@ -669,21 +671,6 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path return false; } -static std::string normalize(const std::string & log_level) -{ - std::string norm = Poco::toLower(log_level); - // normalize - // info -> information - // warn -> warning - if (norm == "info") - return "information"; - else if (norm == "warn") - return "warning"; - else - return norm; -} - - void BaseDaemon::reloadConfiguration() { // when config-file is not specified and config.toml does not exist, we do not load config. @@ -758,7 +745,7 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) // Split log, error log and tracing log. Poco::AutoPtr split = new Poco::ReloadableSplitterChannel; - auto log_level = normalize(config.getString("logger.level", "info")); + auto log_level = Utils::normalizeLogLevel(config.getString("logger.level", "info")); const auto log_path = config.getString("logger.log", ""); if (!log_path.empty()) { diff --git a/metrics/grafana/tiflash_proxy_details.json b/metrics/grafana/tiflash_proxy_details.json index f33fdb0c0df..72bc556b24a 100644 --- a/metrics/grafana/tiflash_proxy_details.json +++ b/metrics/grafana/tiflash_proxy_details.json @@ -14,7 +14,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "7.5.11" + "version": "7.5.17" }, { "type": "panel", @@ -52,7 +52,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1670499325053, + "iteration": 1742544258476, "links": [], "panels": [ { @@ -113,7 +113,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -220,7 +220,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -327,7 +327,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -434,7 +434,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -541,7 +541,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -662,7 +662,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -725,38 +725,1641 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The memory usage of raft entry cache per TiFlash instance", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4560, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "((\n tiflash_proxy_tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{name}}", + "metric": "", + "query": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "((\n tiflash_proxy_raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-raft-engine", + "metric": "", + "query": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory trace", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + } + ], + "repeat": null, + "title": "Cluster", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 4558, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The count of operations per second", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4538, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read_entry {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read_message {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "C", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Operation", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time used in write operation", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4540, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [ + { + "alias": "/^count/", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "/^avg/", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99.99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "(sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "D", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Duration", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The I/O flow rate", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4554, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "rewrite-{{type}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Flow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "binBps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "99% duration breakdown of write operation", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4556, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "wait {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "wal {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "apply {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "C", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Duration Breakdown (99%)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The bytes per write", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4550, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [ + { + "alias": "/^count/", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "/^avg/", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99.99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "(sum(rate(\n tiflash_proxy_raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by () ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "D", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes / Written", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "999% duration breakdown of WAL write operation", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4552, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "sync {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "allocate {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "rotate {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "D", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL Duration Breakdown (999%)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The average number of files", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4546, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "avg((\n tiflash_proxy_raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}} {{$additional_groupby}}", + "metric": "", + "query": "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "avg((\n tiflash_proxy_raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "swap {{$additional_groupby}}", + "metric": "", + "query": "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "avg((\n tiflash_proxy_raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}-recycle {{$additional_groupby}}", + "metric": "", + "query": "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", + "refId": "C", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "File Count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The 99% duration of operations other than write", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4548, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read_entry {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read_message {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "purge {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "C", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Durations (99%)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The average number of log entries", "editable": true, "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, - "y": 25 + "y": 30 }, + "height": null, "hiddenSeries": false, - "id": 4536, + "hideTimeOverride": false, + "id": 4542, + "interval": null, + "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, - "sort": "current", + "sort": "max", "sortDesc": true, "total": false, "values": true @@ -764,42 +2367,54 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, + "repeatDirection": null, "seriesOverrides": [], "spaceLength": 10, + "span": null, "stack": false, "steppedLine": false, "targets": [ { + "datasource": "${DS_TEST-CLUSTER}", "exemplar": true, - "expr": "tiflash_proxy_tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}", + "expr": "avg((\n tiflash_proxy_raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", + "hide": false, + "instant": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}} {{instance}}", + "intervalFactor": 1, + "legendFormat": "{{type}} {{$additional_groupby}}", + "metric": "", + "query": "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "A", - "step": 10 + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raft Entry Cache", + "title": "Entry Count", "tooltip": { - "msResolution": false, + "msResolution": true, "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, + "transformations": [], "type": "graph", "xaxis": { "buckets": null, @@ -810,14 +2425,16 @@ }, "yaxes": [ { - "format": "bytes", + "decimals": null, + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { + "decimals": null, "format": "short", "label": null, "logBase": 1, @@ -828,12 +2445,218 @@ ], "yaxis": { "align": false, - "alignLevel": null + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The compression ratio per write", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "height": null, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 4544, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [ + { + "alias": "/^count/", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "/^avg/", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], + "spaceLength": 10, + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99.99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99% {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "(sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "exemplar": true, + "expr": "sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", + "refId": "D", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write Compression Ratio", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 } } ], - "repeat": null, - "title": "Cluster", + "title": "Raft Engine", "type": "row" }, { @@ -843,7 +2666,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 2 }, "id": 2743, "panels": [ @@ -1864,7 +3687,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 3 }, "id": 2744, "panels": [ @@ -3102,7 +4925,7 @@ "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 4 }, "id": 2745, "panels": [ @@ -3715,7 +5538,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 5 }, "id": 2746, "panels": [ @@ -5275,7 +7098,7 @@ "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 6 }, "id": 2747, "panels": [ @@ -5287,13 +7110,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of requests that TiKV sends to PD", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 7 }, + "hiddenSeries": false, "id": 1069, "legend": { "alignAsTable": true, @@ -5313,7 +7142,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5380,13 +7213,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed by requests that TiKV sends to PD", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 7 }, + "hiddenSeries": false, "id": 1070, "legend": { "alignAsTable": true, @@ -5406,7 +7245,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5473,13 +7316,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": " \tThe total number of PD heartbeat messages", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 15 }, + "hiddenSeries": false, "id": 1215, "legend": { "alignAsTable": true, @@ -5499,7 +7348,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5566,13 +7419,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The total number of peers validated by the PD worker", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 15 }, + "hiddenSeries": false, "id": 1396, "legend": { "alignAsTable": true, @@ -5592,7 +7451,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5664,7 +7527,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 2748, "panels": [ @@ -5677,14 +7540,20 @@ "description": "The time consumed when Raft applies log", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 7 + "y": 8 }, + "hiddenSeries": false, "id": 31, "legend": { "alignAsTable": true, @@ -5703,7 +7572,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5789,14 +7662,20 @@ "description": "The time consumed for Raft to apply logs per TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 7 + "y": 8 }, + "hiddenSeries": false, "id": 32, "legend": { "alignAsTable": true, @@ -5815,7 +7694,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5884,14 +7767,20 @@ "description": "The time consumed when Raft appends log", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 15 }, + "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, @@ -5910,7 +7799,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -5996,14 +7889,20 @@ "description": "The time consumed when Raft appends log on each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 15 }, + "hiddenSeries": false, "id": 40, "legend": { "alignAsTable": true, @@ -6022,7 +7921,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6089,13 +7992,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 22 }, + "hiddenSeries": false, "id": 3690, "legend": { "alignAsTable": true, @@ -6112,7 +8021,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -6191,13 +8104,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log on each TiKV instance", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 22 }, + "hiddenSeries": false, "id": 3688, "legend": { "alignAsTable": true, @@ -6214,7 +8133,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -6284,7 +8207,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 8 }, "id": 2749, "panels": [ @@ -6298,14 +8221,20 @@ "description": "The count of different ready type of Raft", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, + "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": true, @@ -6325,7 +8254,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6404,14 +8337,20 @@ "description": "The time consumed for peer processes to be ready in Raft", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, + "hiddenSeries": false, "id": 118, "legend": { "alignAsTable": true, @@ -6431,7 +8370,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6509,14 +8452,20 @@ "description": "The time consumed by raftstore events (P99).99", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, + "hiddenSeries": false, "id": 123, "legend": { "alignAsTable": true, @@ -6536,7 +8485,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6616,7 +8569,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 9 }, "id": 2750, "panels": [ @@ -6630,14 +8583,20 @@ "description": "The number of Raft messages sent by each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 10 }, + "hiddenSeries": false, "id": 1615, "legend": { "alignAsTable": true, @@ -6657,7 +8616,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6727,14 +8690,20 @@ "description": "The number of Raft messages flushed by each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 47 + "y": 10 }, + "hiddenSeries": false, "id": 1616, "legend": { "alignAsTable": true, @@ -6754,7 +8723,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6823,13 +8796,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of Raft messages received by each TiKV instance", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 54 + "y": 17 }, + "hiddenSeries": false, "id": 106, "legend": { "alignAsTable": true, @@ -6849,7 +8828,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -6918,14 +8901,20 @@ "description": "The number of different types of Raft messages that are sent", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 54 + "y": 17 }, + "hiddenSeries": false, "id": 11, "legend": { "alignAsTable": true, @@ -6945,7 +8934,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7015,14 +9008,20 @@ "description": "The total number of vote messages that are sent in Raft", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 61 + "y": 24 }, + "hiddenSeries": false, "id": 25, "legend": { "alignAsTable": true, @@ -7042,7 +9041,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7112,14 +9115,20 @@ "description": "The number of dropped Raft messages per type", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 61 + "y": 24 }, + "hiddenSeries": false, "id": 1309, "legend": { "alignAsTable": true, @@ -7139,7 +9148,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7211,7 +9224,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 10 }, "id": 2751, "panels": [ @@ -7223,13 +9236,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The proposal count of all Regions in a mio tick", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 48 + "y": 11 }, + "hiddenSeries": false, "id": 108, "legend": { "alignAsTable": true, @@ -7249,7 +9268,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7319,14 +9342,20 @@ "description": "The number of proposals per type", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 48 + "y": 11 }, + "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, @@ -7346,7 +9375,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7417,14 +9450,20 @@ "description": "The number of read proposals which are made by each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 55 + "y": 18 }, + "hiddenSeries": false, "id": 119, "legend": { "alignAsTable": true, @@ -7444,7 +9483,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7515,14 +9558,20 @@ "description": "The number of write proposals which are made by each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 55 + "y": 18 }, + "hiddenSeries": false, "id": 120, "legend": { "alignAsTable": true, @@ -7542,7 +9591,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7612,14 +9665,20 @@ "description": "The wait time of each proposal", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 62 + "y": 25 }, + "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": true, @@ -7638,7 +9697,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7724,14 +9787,20 @@ "description": "The wait time of each proposal in each TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 62 + "y": 25 }, + "hiddenSeries": false, "id": 42, "legend": { "alignAsTable": true, @@ -7750,7 +9819,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7818,14 +9891,20 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 69 + "y": 32 }, + "hiddenSeries": false, "id": 2535, "legend": { "alignAsTable": true, @@ -7844,7 +9923,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -7929,14 +10012,20 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 69 + "y": 32 }, + "hiddenSeries": false, "id": 2536, "legend": { "alignAsTable": true, @@ -7955,7 +10044,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8022,13 +10115,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 76 + "y": 39 }, + "hiddenSeries": false, "id": 1975, "legend": { "alignAsTable": true, @@ -8047,7 +10146,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8114,14 +10217,20 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 76 + "y": 39 }, + "hiddenSeries": false, "id": 4375, "legend": { "alignAsTable": true, @@ -8140,7 +10249,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8221,7 +10334,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 2752, "panels": [ @@ -8235,14 +10348,20 @@ "description": "The number of admin proposals", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 11 + "y": 12 }, + "hiddenSeries": false, "id": 76, "legend": { "alignAsTable": true, @@ -8262,7 +10381,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8333,14 +10456,20 @@ "description": "The number of the processed apply command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 11 + "y": 12 }, + "hiddenSeries": false, "id": 77, "legend": { "alignAsTable": true, @@ -8360,7 +10489,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8431,14 +10564,20 @@ "description": "The number of raftstore split checksss", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 18 + "y": 19 }, + "hiddenSeries": false, "id": 70, "legend": { "alignAsTable": true, @@ -8458,7 +10597,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8529,14 +10672,20 @@ "description": "The time consumed when running split check in .9999", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 18 + "y": 19 }, + "hiddenSeries": false, "id": 71, "legend": { "alignAsTable": true, @@ -8557,7 +10706,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -8630,7 +10783,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 4200, "panels": [ @@ -8641,13 +10794,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used by each level in the unified read pool per second. Level 0 refers to small queries.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "hiddenSeries": false, "id": 4194, "legend": { "alignAsTable": true, @@ -8665,7 +10824,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -8730,13 +10893,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The chance that level 0 (small) tasks are scheduled in the unified read pool.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "hiddenSeries": false, "id": 4196, "legend": { "alignAsTable": true, @@ -8753,7 +10922,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -8818,13 +10991,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of concurrently running tasks in the unified read pool.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, + "hiddenSeries": false, "id": 4198, "legend": { "alignAsTable": true, @@ -8841,7 +11020,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": false, "renderer": "flot", @@ -8911,7 +11094,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 2754, "panels": [ @@ -8925,7 +11108,12 @@ "description": "The total count of different kinds of commands received", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -8933,6 +11121,7 @@ "x": 0, "y": 14 }, + "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, @@ -8954,7 +11143,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9024,7 +11217,12 @@ "description": "The total number of engine asynchronous request errors", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9032,6 +11230,7 @@ "x": 12, "y": 14 }, + "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, @@ -9053,7 +11252,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9124,7 +11327,12 @@ "description": "The time consumed by processing asynchronous snapshot requests", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9132,6 +11340,7 @@ "x": 0, "y": 22 }, + "hiddenSeries": false, "id": 15, "legend": { "alignAsTable": true, @@ -9152,7 +11361,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9238,7 +11451,12 @@ "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9246,6 +11464,7 @@ "x": 12, "y": 22 }, + "hiddenSeries": false, "id": 109, "legend": { "alignAsTable": true, @@ -9266,7 +11485,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9354,7 +11577,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 2755, "panels": [ @@ -9366,7 +11589,12 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The total number of commands on each stage", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 10, "w": 12, @@ -9374,6 +11602,7 @@ "y": 15 }, "height": "400", + "hiddenSeries": false, "id": 167, "legend": { "alignAsTable": true, @@ -9394,7 +11623,11 @@ "links": [], "maxPerRow": 1, "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9469,7 +11702,12 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The total writing bytes of commands on each stage", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 10, "w": 12, @@ -9477,6 +11715,7 @@ "y": 15 }, "height": "400", + "hiddenSeries": false, "id": 3834, "legend": { "alignAsTable": true, @@ -9497,7 +11736,11 @@ "links": [], "maxPerRow": 1, "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9566,7 +11809,12 @@ "description": "The count of different priority commands", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9575,6 +11823,7 @@ "y": 25 }, "height": "", + "hiddenSeries": false, "id": 1, "legend": { "alignAsTable": true, @@ -9595,7 +11844,11 @@ "links": [], "maxPerRow": 2, "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9700,7 +11953,12 @@ "description": "The count of pending commands per TiKV instance", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9709,6 +11967,7 @@ "y": 25 }, "height": "", + "hiddenSeries": false, "id": 193, "legend": { "alignAsTable": true, @@ -9729,7 +11988,11 @@ "links": [], "maxPerRow": 2, "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9754,7 +12017,8 @@ "fill": true, "line": true, "op": "gt", - "value": 300 + "value": 300, + "visible": true } ], "timeFrom": null, @@ -9810,7 +12074,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 2756, "panels": [ @@ -9822,7 +12086,12 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The total number of commands on each stage in commit command", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 10, "w": 24, @@ -9830,6 +12099,7 @@ "y": 16 }, "height": "400", + "hiddenSeries": false, "id": 168, "legend": { "alignAsTable": true, @@ -9850,7 +12120,11 @@ "links": [], "maxPerRow": 1, "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -9934,7 +12208,12 @@ "description": "The time consumed when executing commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -9942,6 +12221,7 @@ "x": 0, "y": 26 }, + "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": true, @@ -9963,7 +12243,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10059,7 +12343,12 @@ "description": "The time which is caused by latch wait in commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -10067,6 +12356,7 @@ "x": 12, "y": 26 }, + "hiddenSeries": false, "id": 194, "legend": { "alignAsTable": true, @@ -10088,7 +12378,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10184,7 +12478,12 @@ "description": "The count of keys read by a commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -10192,6 +12491,7 @@ "x": 0, "y": 34 }, + "hiddenSeries": false, "id": 195, "legend": { "alignAsTable": true, @@ -10213,7 +12513,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10309,7 +12613,12 @@ "description": "The count of keys written by a commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -10317,6 +12626,7 @@ "x": 12, "y": 34 }, + "hiddenSeries": false, "id": 373, "legend": { "alignAsTable": true, @@ -10338,7 +12648,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10434,6 +12748,10 @@ "description": "The keys scan details of each CF when executing commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "grid": {}, "gridPos": { @@ -10464,6 +12782,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10541,6 +12860,10 @@ "description": "The keys scan details of lock CF when executing commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "grid": {}, "gridPos": { @@ -10571,6 +12894,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10648,6 +12972,10 @@ "description": "The keys scan details of write CF when executing commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "grid": {}, "gridPos": { @@ -10678,6 +13006,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10755,6 +13084,10 @@ "description": "The keys scan details of default CF when executing commit command", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "grid": {}, "gridPos": { @@ -10785,6 +13118,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10864,7 +13198,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 2759, "panels": [ @@ -10877,14 +13211,20 @@ "description": "The rate of Raft snapshot messages sent", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 16 + "y": 17 }, + "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, @@ -10904,7 +13244,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -10973,14 +13317,20 @@ "description": "The time consumed when handling snapshots", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 16 + "y": 17 }, + "hiddenSeries": false, "id": 36, "legend": { "alignAsTable": true, @@ -11000,7 +13350,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11085,14 +13439,20 @@ "description": "The number of snapshots in different states", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 16 + "y": 17 }, + "hiddenSeries": false, "id": 38, "legend": { "alignAsTable": true, @@ -11112,7 +13472,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11182,14 +13546,20 @@ "description": "The snapshot size (P99.99).9999", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 23 + "y": 24 }, + "hiddenSeries": false, "id": 44, "legend": { "alignAsTable": true, @@ -11209,7 +13579,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11279,14 +13653,20 @@ "description": "The number of KV within a snapshot in .9999", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 23 + "y": 24 }, + "hiddenSeries": false, "id": 43, "legend": { "alignAsTable": true, @@ -11306,7 +13686,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11379,7 +13763,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 2760, "panels": [ @@ -11393,14 +13777,20 @@ "description": "The number of tasks handled by worker", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 18 }, + "hiddenSeries": false, "id": 59, "legend": { "alignAsTable": true, @@ -11422,7 +13812,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11492,14 +13886,20 @@ "description": " \tCurrent pending and running tasks of worker", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 18 }, + "hiddenSeries": false, "id": 1395, "legend": { "alignAsTable": true, @@ -11521,7 +13921,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11591,14 +13995,20 @@ "description": "The number of tasks handled by future_pool", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 26 }, + "hiddenSeries": false, "id": 1876, "legend": { "alignAsTable": true, @@ -11620,7 +14030,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11690,14 +14104,20 @@ "description": "Current pending and running tasks of future_pool", "editable": true, "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 26 }, + "hiddenSeries": false, "id": 1877, "legend": { "alignAsTable": true, @@ -11719,7 +14139,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -11791,7 +14215,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 2761, "panels": [ @@ -11802,13 +14226,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 24 + "y": 19 }, + "hiddenSeries": false, "id": 2108, "legend": { "alignAsTable": true, @@ -11828,7 +14258,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": true, "renderer": "flot", @@ -11901,13 +14335,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 24 + "y": 19 }, + "hiddenSeries": false, "id": 2258, "legend": { "alignAsTable": true, @@ -11927,7 +14367,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": true, "renderer": "flot", @@ -11995,13 +14439,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 31 + "y": 26 }, + "hiddenSeries": false, "id": 2660, "legend": { "alignAsTable": true, @@ -12021,7 +14471,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": true, "renderer": "flot", @@ -12089,13 +14543,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 31 + "y": 26 }, + "hiddenSeries": false, "id": 2661, "legend": { "alignAsTable": true, @@ -12115,7 +14575,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 2, "points": true, "renderer": "flot", @@ -12188,7 +14652,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 2762, "panels": [ @@ -12200,13 +14664,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of get operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 20 }, + "hiddenSeries": false, "id": 138, "legend": { "alignAsTable": true, @@ -12226,7 +14696,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12334,13 +14808,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing get operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 20 }, + "hiddenSeries": false, "id": 82, "legend": { "alignAsTable": true, @@ -12360,7 +14840,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12458,13 +14942,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of seek operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 28 }, + "hiddenSeries": false, "id": 129, "legend": { "alignAsTable": true, @@ -12484,7 +14974,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12604,13 +15098,19 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing seek operation", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 28 }, + "hiddenSeries": false, "id": 125, "legend": { "alignAsTable": true, @@ -12630,7 +15130,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12728,14 +15232,146 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of write operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 139, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "done", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "timeout", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "with_wal", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Write operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The time consumed when executing write operation", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 36 }, - "id": 139, + "hiddenSeries": false, + "id": 126, "legend": { "alignAsTable": true, "avg": false, @@ -12754,7 +15390,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12771,35 +15411,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", + "expr": "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "done", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "timeout", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))", + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "with_wal", + "legendFormat": "95%", "refId": "C", "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write operations", + "title": "Write duration", "tooltip": { "shared": true, "sort": 0, @@ -12815,9 +15463,9 @@ }, "yaxes": [ { - "format": "ops", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -12843,15 +15491,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The time consumed when executing write operation", - "fill": 0, + "description": " \tThe count of WAL sync operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 44 }, - "id": 126, + "id": 137, "legend": { "alignAsTable": true, "avg": false, @@ -12871,6 +15523,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -12887,43 +15540,20 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", + "expr": "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "max", + "legendFormat": "sync", + "metric": "", "refId": "A", "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "D", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write duration", + "title": "WAL sync operations", "tooltip": { "shared": true, "sort": 0, @@ -12939,9 +15569,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "ops", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -12968,12 +15598,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing write wal operation", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 44 }, "id": 130, "legend": { @@ -12995,6 +15629,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13091,15 +15726,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": " \tThe count of WAL sync operations", + "description": "The count of compaction and flush operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 52 }, - "id": 137, + "id": 128, "legend": { "alignAsTable": true, "avg": false, @@ -13119,6 +15758,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13135,12 +15775,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "sync", - "metric": "", - "refId": "A", + "legendFormat": "{{type}}", + "metric": "tiflash_proxy_tikv_engine_event_total", + "refId": "B", "step": 10 } ], @@ -13148,7 +15788,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "WAL sync operations", + "title": "Compaction operations", "tooltip": { "shared": true, "sort": 0, @@ -13193,12 +15833,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing WAL sync operation", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 49 + "y": 52 }, "id": 135, "legend": { @@ -13221,6 +15865,7 @@ "maxPerRow": 2, "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13317,15 +15962,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The count of compaction and flush operations", + "description": "The time consumed when reading SST files", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 60 }, - "id": 128, + "id": 140, "legend": { "alignAsTable": true, "avg": false, @@ -13345,6 +15994,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13361,20 +16011,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", + "expr": "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tiflash_proxy_tikv_engine_event_total", + "legendFormat": "max", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", "refId": "B", "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Compaction operations", + "title": "SST read duration", "tooltip": { "shared": true, "sort": 0, @@ -13390,9 +16067,9 @@ }, "yaxes": [ { - "format": "ops", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -13419,12 +16096,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing the compaction and flush operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 60 }, "id": 136, "legend": { @@ -13446,6 +16127,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13543,15 +16225,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The time consumed when reading SST files", + "description": "The block cache size. Broken down by column family if shared block cache is disabled.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 68 }, - "id": 140, + "id": 102, "legend": { "alignAsTable": true, "avg": false, @@ -13571,6 +16257,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13587,47 +16274,19 @@ "steppedLine": false, "targets": [ { - "expr": "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", + "expr": "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "max", - "metric": "", + "legendFormat": "{{instance}}-{{cf}}", "refId": "A", "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "avg", - "metric": "", - "refId": "D", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "SST read duration", + "title": "Block cache size", "tooltip": { "shared": true, "sort": 0, @@ -13643,9 +16302,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "bytes", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -13671,15 +16330,18 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The time which is caused by write stall", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 153 + "y": 68 }, - "id": 87, + "id": 2451, "legend": { "alignAsTable": true, "avg": false, @@ -13699,55 +16361,30 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max", - "metric": "", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "95%", - "metric": "", - "refId": "C", - "step": 10 - }, + "scopedVars": { + "db": { + "selected": false, + "text": "kv", + "value": "kv" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "avg", + "legendFormat": "{{cf}} - {{reason}}", "metric": "", - "refId": "D", + "refId": "A", "step": 10 } ], @@ -13755,7 +16392,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write stall duration", + "title": "Compaction reason", "tooltip": { "shared": true, "sort": 0, @@ -13771,11 +16408,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -13783,7 +16420,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -13799,15 +16436,20 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The memtable size of each column family", - "fill": 1, + "description": "The flow of different kinds of block cache operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 153 + "x": 0, + "y": 76 }, - "id": 103, + "height": "", + "id": 467, "legend": { "alignAsTable": true, "avg": false, @@ -13827,6 +16469,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13843,19 +16486,86 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{cf}}", + "legendFormat": "total_read", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total_written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "data_insert", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_insert", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_evict", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_insert", + "metric": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_evict", + "metric": "", + "refId": "G", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memtable size", + "title": "Block cache flow", "tooltip": { "shared": true, "sort": 0, @@ -13871,19 +16581,19 @@ }, "yaxes": [ { - "format": "bytes", + "format": "Bps", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -13900,12 +16610,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The hit rate of memtable", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 73 + "y": 76 }, "id": 88, "legend": { @@ -13927,6 +16641,7 @@ "links": [], "nullPointMode": "connected", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -13999,15 +16714,20 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The block cache size. Broken down by column family if shared block cache is disabled.", - "fill": 1, + "description": "The flow of different kinds of operations on keys", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 81 + "y": 84 }, - "id": 102, + "height": "", + "id": 132, "legend": { "alignAsTable": true, "avg": false, @@ -14027,6 +16747,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14043,10 +16764,33 @@ "steppedLine": false, "targets": [ { - "expr": "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}", + "legendFormat": "read", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "corrupt", + "metric": "", "refId": "A", "step": 10 } @@ -14055,7 +16799,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Block cache size", + "title": "Keys flow", "tooltip": { "shared": true, "sort": 0, @@ -14071,11 +16815,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -14083,7 +16827,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -14100,12 +16844,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The hit rate of block cache", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 81 + "y": 84 }, "id": 80, "legend": { @@ -14128,6 +16876,7 @@ "maxPerRow": 2, "nullPointMode": "connected", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14237,16 +16986,20 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The flow of different kinds of block cache operations", + "description": "The flow rate of read operations per type", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 92 }, "height": "", - "id": 467, + "id": 85, "legend": { "alignAsTable": true, "avg": false, @@ -14266,6 +17019,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14282,86 +17036,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "total_read", + "legendFormat": "get", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "total_written", + "legendFormat": "scan", "refId": "C", "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "data_insert", - "metric": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_insert", - "metric": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "filter_evict", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_insert", - "metric": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "index_evict", - "metric": "", - "refId": "G", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Block cache flow", + "title": "Read flow", "tooltip": { "shared": true, "sort": 0, @@ -14379,13 +17078,13 @@ { "format": "Bps", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": "0", "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -14406,12 +17105,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of different kinds of block cache operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 92 }, "id": 468, "legend": { @@ -14433,6 +17136,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14542,16 +17246,20 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The flow of different kinds of operations on keys", - "fill": 0, + "description": "The flow of different kinds of write operations", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 100 }, "height": "", - "id": 132, + "id": 86, "legend": { "alignAsTable": true, "avg": false, @@ -14571,6 +17279,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14587,33 +17296,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "read", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "written", + "legendFormat": "wal", "refId": "C", "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "corrupt", - "metric": "", + "legendFormat": "write", "refId": "A", "step": 10 } @@ -14622,7 +17318,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Keys flow", + "title": "Write flow", "tooltip": { "shared": true, "sort": 0, @@ -14638,7 +17334,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -14667,12 +17363,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The count of keys in each column family", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 100 }, "id": 131, "legend": { @@ -14694,6 +17394,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14768,16 +17469,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The flow rate of read operations per type", - "fill": 0, + "description": "The flow rate of compaction operations per type", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 108 }, - "height": "", - "id": 85, + "id": 90, "legend": { "alignAsTable": true, "avg": false, @@ -14797,6 +17501,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -14813,31 +17518,38 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "get", + "legendFormat": "read", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "scan", + "legendFormat": "written", "refId": "C", "step": 10 + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "flushed", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read flow", + "title": "Compaction flow", "tooltip": { "shared": true, "sort": 0, @@ -14861,7 +17573,7 @@ "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -14882,12 +17594,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The bytes per read", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 105 + "y": 108 }, "id": 133, "legend": { @@ -14910,6 +17626,7 @@ "maxPerRow": 2, "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15006,16 +17723,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The flow of different kinds of write operations", + "description": "The read amplification per TiKV instance \t", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 113 + "y": 116 }, - "height": "", - "id": 86, + "id": 518, "legend": { "alignAsTable": true, "avg": false, @@ -15035,6 +17755,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15051,20 +17772,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "wal", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", + "expr": "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "write", + "legendFormat": "{{instance}}", + "metric": "", "refId": "A", "step": 10 } @@ -15073,7 +17786,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write flow", + "title": "Read amplication", "tooltip": { "shared": true, "sort": 0, @@ -15089,7 +17802,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -15118,12 +17831,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The bytes per write", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 113 + "y": 116 }, "id": 134, "legend": { @@ -15146,6 +17863,7 @@ "maxPerRow": 2, "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15242,15 +17960,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The flow rate of compaction operations per type", + "description": "The number of snapshot of each TiKV instance", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 121 + "y": 124 }, - "id": 90, + "id": 516, "legend": { "alignAsTable": true, "avg": false, @@ -15270,6 +17992,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15286,38 +18009,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", + "expr": "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "{{instance}}", + "metric": "", "refId": "A", "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "written", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "flushed", - "refId": "B", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Compaction flow", + "title": "Number of snapshots", "tooltip": { "shared": true, "sort": 0, @@ -15333,7 +18039,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -15341,7 +18047,7 @@ "show": true }, { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -15362,12 +18068,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The pending bytes to be compacted", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 121 + "y": 124 }, "id": 127, "legend": { @@ -15389,6 +18099,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15443,109 +18154,7 @@ "show": true }, { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The read amplification per TiKV instance \t", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 129 - }, - "id": 518, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "scopedVars": { - "db": { - "selected": false, - "text": "kv", - "value": "kv" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Read amplication", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -15564,25 +18173,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The compression ratio of each level", + "description": "The number of SST files for different column families in each level", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 129 + "x": 0, + "y": 132 }, - "id": 863, + "id": 2002, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, - "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -15593,6 +18204,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15609,21 +18221,18 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)", + "expr": "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "level - {{level}}", - "metric": "", - "refId": "A", - "step": 10 + "legendFormat": "cf-{{cf}}, level-{{level}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Compression ratio", + "title": "Number files at each level", "tooltip": { "shared": true, "sort": 0, @@ -15643,7 +18252,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15651,7 +18260,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -15667,15 +18276,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The number of snapshot of each TiKV instance", + "description": "The compression ratio of each level", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 137 + "x": 12, + "y": 132 }, - "id": 516, + "id": 863, "legend": { "alignAsTable": true, "avg": false, @@ -15695,6 +18308,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15711,11 +18325,11 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}", + "legendFormat": "level - {{level}}", "metric": "", "refId": "A", "step": 10 @@ -15725,7 +18339,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number of snapshots", + "title": "Compression ratio", "tooltip": { "shared": true, "sort": 0, @@ -15768,25 +18382,28 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The time that the oldest unreleased snapshot survivals", + "description": "Stall conditions changed of each column family", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 137 + "x": 0, + "y": 140 }, - "id": 517, + "id": 2381, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideZero": true, "max": true, - "min": false, + "min": true, "rightSide": true, "show": true, - "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -15797,6 +18414,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15813,21 +18431,18 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "tiflash_proxy_tikv_engine_oldest_snapshot_duration", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}}-{{cf}}-{{type}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Oldest snapshots duration", + "title": "Stall conditions changed of each CF", "tooltip": { "shared": true, "sort": 0, @@ -15843,11 +18458,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -15855,7 +18470,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -15870,23 +18485,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of SST files for different column families in each level", + "decimals": 1, + "description": "The time that the oldest unreleased snapshot survivals", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 145 + "x": 12, + "y": 140 }, - "id": 2002, + "id": 517, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -15897,6 +18518,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -15913,18 +18535,21 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)", + "expr": "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "cf-{{cf}}, level-{{level}}", - "refId": "A" + "legendFormat": "{{instance}}", + "metric": "tiflash_proxy_tikv_engine_oldest_snapshot_duration", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number files at each level", + "title": "Oldest snapshots duration", "tooltip": { "shared": true, "sort": 0, @@ -15940,11 +18565,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -15952,7 +18577,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -15967,23 +18592,28 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed when ingesting SST files", + "decimals": 1, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 145 + "x": 0, + "y": 148 }, - "id": 2003, + "id": 2452, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, - "min": true, + "min": false, "rightSide": true, "show": true, + "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, @@ -15994,6 +18624,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16010,25 +18641,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A" - }, - { - "expr": "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "average", - "refId": "B" + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Ingest SST duration seconds", + "title": "Write Stall Reason", "tooltip": { "shared": true, "sort": 0, @@ -16044,11 +18671,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16056,7 +18683,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -16071,20 +18698,23 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Stall conditions changed of each column family", + "description": "The time consumed when ingesting SST files", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 153 + "x": 12, + "y": 148 }, - "id": 2381, + "id": 2003, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideZero": true, "max": true, "min": true, "rightSide": true, @@ -16099,6 +18729,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16115,10 +18746,17 @@ "steppedLine": false, "targets": [ { - "expr": "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}", + "expr": "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}-{{type}}", + "legendFormat": "99%", + "refId": "A" + }, + { + "expr": "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average", "refId": "B" } ], @@ -16126,7 +18764,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Stall conditions changed of each CF", + "title": "Ingest SST duration seconds", "tooltip": { "shared": true, "sort": 0, @@ -16142,7 +18780,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -16170,14 +18808,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "description": "The time which is caused by write stall", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 161 + "x": 12, + "y": 156 }, - "id": 2452, + "id": 87, "legend": { "alignAsTable": true, "avg": false, @@ -16197,6 +18840,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16213,21 +18857,47 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)", + "expr": "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "max", "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write Stall Reason", + "title": "Write stall duration", "tooltip": { "shared": true, "sort": 0, @@ -16243,11 +18913,11 @@ }, "yaxes": [ { - "format": "short", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16255,7 +18925,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -16271,14 +18941,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, + "description": "The memtable size of each column family", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 164 }, - "id": 2451, + "id": 103, "legend": { "alignAsTable": true, "avg": false, @@ -16298,6 +18973,7 @@ "links": [], "nullPointMode": "null", "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -16314,12 +18990,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)", + "expr": "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{cf}} - {{reason}}", - "metric": "", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 } @@ -16328,7 +19002,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Compaction reason", + "title": "Memtable size", "tooltip": { "shared": true, "sort": 0, @@ -16344,11 +19018,11 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -16356,7 +19030,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -16845,7 +19519,7 @@ } ], "refresh": "1m", - "schemaVersion": 18, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -16984,6 +19658,37 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "none", + "value": "none" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "additional_groupby", + "multi": false, + "name": "additional_groupby", + "options": [ + { + "selected": true, + "text": "none", + "value": "none" + }, + { + "selected": false, + "text": "instance", + "value": "instance" + } + ], + "query": "none,instance", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" } ] }, diff --git a/metrics/grafana/tiflash_summary.json b/metrics/grafana/tiflash_summary.json index 99f12b7a7aa..05525eefde0 100644 --- a/metrics/grafana/tiflash_summary.json +++ b/metrics/grafana/tiflash_summary.json @@ -18884,4 +18884,4 @@ "title": "Test-Cluster-TiFlash-Summary", "uid": "SVbh2xUWk", "version": 1 -} \ No newline at end of file +}