Storage: Fix TableScan performance regression under wide-sparse table (#10379)

dbms/src/IO/Checksum/ChecksumBuffer.h

@@ -303,8 +303,11 @@ class FramedChecksumReadBuffer : public ReadBufferFromFileDescriptor
                 if (unlikely(frame.checksum != digest.checksum()))
                 {
                     throw TiFlashException(
-                        "checksum mismatch for " + in->getFileName(),
-                        Errors::Checksum::DataCorruption);
+                        Errors::Checksum::DataCorruption,
+                        "checksum mismatch, file={} expect={:x} actual={:x}",
+                        in->getFileName(),
+                        frame.checksum,
+                        digest.checksum());
                 }
             }
 
@@ -376,7 +379,12 @@ class FramedChecksumReadBuffer : public ReadBufferFromFileDescriptor
             digest.update(frame.data, frame.bytes);
             if (unlikely(frame.checksum != digest.checksum()))
             {
-                throw TiFlashException("checksum mismatch for " + in->getFileName(), Errors::Checksum::DataCorruption);
+                throw TiFlashException(
+                    Errors::Checksum::DataCorruption,
+                    "checksum mismatch, file={} expect={:x} actual={:x}",
+                    in->getFileName(),
+                    frame.checksum,
+                    digest.checksum());
             }
         }
 

dbms/src/IO/FileProvider/ChecksumReadBufferBuilder.cpp

@@ -55,12 +55,11 @@ std::unique_ptr<ReadBufferFromFileBase> ChecksumReadBufferBuilder::build(
 std::unique_ptr<ReadBufferFromFileBase> ChecksumReadBufferBuilder::build(
     String && data,
     const String & file_name,
-    size_t estimated_size,
     ChecksumAlgo checksum_algorithm,
     size_t checksum_frame_size)
 {
+    auto allocation_size = std::min(data.size(), checksum_frame_size);
     auto file = std::make_shared<MemoryRandomAccessFile>(file_name, std::forward<String>(data));
-    auto allocation_size = std::min(estimated_size, checksum_frame_size);
     switch (checksum_algorithm)
     {
     case ChecksumAlgo::None:

dbms/src/IO/FileProvider/ChecksumReadBufferBuilder.h

@@ -40,7 +40,6 @@ class ChecksumReadBufferBuilder
     static std::unique_ptr<ReadBufferFromFileBase> build(
         String && data,
         const String & file_name,
-        size_t estimated_size,
         ChecksumAlgo checksum_algorithm,
         size_t checksum_frame_size);
 };

dbms/src/IO/FileProvider/CompressedReadBufferFromFileBuilder.cpp

@@ -40,16 +40,11 @@ std::unique_ptr<LegacyCompressedReadBufferFromFile> CompressedReadBufferFromFile
 std::unique_ptr<CompressedReadBufferFromFile> CompressedReadBufferFromFileBuilder::build(
     String && data,
     const String & file_name,
-    size_t estimated_size,
     ChecksumAlgo checksum_algorithm,
     size_t checksum_frame_size)
 {
-    auto file_in = ChecksumReadBufferBuilder::build(
-        std::move(data),
-        file_name,
-        estimated_size,
-        checksum_algorithm,
-        checksum_frame_size);
+    auto file_in
+        = ChecksumReadBufferBuilder::build(std::move(data), file_name, checksum_algorithm, checksum_frame_size);
     return std::make_unique<CompressedReadBufferFromFileImpl<false>>(std::move(file_in));
 }
 
@@ -73,4 +68,4 @@ std::unique_ptr<CompressedReadBufferFromFile> CompressedReadBufferFromFileBuilde
     return std::make_unique<CompressedReadBufferFromFileImpl<false>>(std::move(file_in));
 }
 
-} // namespace DB
+} // namespace DB

dbms/src/IO/FileProvider/CompressedReadBufferFromFileBuilder.h

@@ -34,7 +34,6 @@ class CompressedReadBufferFromFileBuilder
     static std::unique_ptr<CompressedReadBufferFromFile> build(
         String && data,
         const String & file_name,
-        size_t estimated_size,
         ChecksumAlgo checksum_algorithm,
         size_t checksum_frame_size);
 

dbms/src/Server/DTTool/DTTool.h

@@ -56,8 +56,11 @@ struct InspectArgs
     bool check;
     bool dump_columns;
     bool dump_all_columns;
+    bool dump_minmax;
+    bool dump_merged_files;
     size_t file_id;
     std::string workdir;
+    std::vector<DB::ColumnID> col_ids;
 };
 int inspectEntry(const std::vector<std::string> & opts, RaftStoreFFIFunc ffi_function);
 } // namespace DTTool::Inspect
@@ -127,10 +130,16 @@ class ImitativeEnv
     static void setupLogger()
     {
         Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cout);
-        Poco::AutoPtr<Poco::Formatter> formatter(new UnifiedLogFormatter<true>());
+        // only enable colors when the output is a terminal
+        bool enable_colors = isatty(STDOUT_FILENO) && isatty(STDERR_FILENO);
+        Poco::AutoPtr<Poco::Formatter> formatter;
+        if (enable_colors)
+            formatter = new UnifiedLogFormatter<true>();
+        else
+            formatter = new UnifiedLogFormatter<false>();
         Poco::AutoPtr<Poco::FormattingChannel> formatting_channel(new Poco::FormattingChannel(formatter, channel));
         Poco::Logger::root().setChannel(formatting_channel);
-        Poco::Logger::root().setLevel("trace");
+        Poco::Logger::root().setLevel("debug");
     }
 
     ContextPtr global_context{};