Add flag for ingest range check (#7767) (#7787)

close #7766
pingcap · Jul 11, 2023 · 338634a · 338634a
1 parent dbf0cf0
commit 338634a
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 9 deletions.
diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h
@@ -228,6 +228,7 @@ struct Settings
     M(SettingChecksumAlgorithm, dt_checksum_algorithm, ChecksumAlgo::XXH3, "Checksum algorithm for delta tree stable storage")                                                                                                          \
     M(SettingCompressionMethod, dt_compression_method, CompressionMethod::LZ4, "The method of data compression when writing.")                                                                                                          \
     M(SettingInt64, dt_compression_level, 1, "The compression level.")                                                                                                                                                                  \
+    M(SettingBool, dt_enable_ingest_check, true, "Check for illegal ranges when ingesting SST files.")                                                                                                                                  \
     \
     M(SettingInt64, remote_checkpoint_interval_seconds, 30, "The interval of uploading checkpoint to the remote store. Unit is second.")                                                                                                \
     M(SettingInt64, remote_gc_method, 1, "The method of running GC task on the remote store. 1 - lifecycle, 2 - scan.")                                                                                                                 \

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp
@@ -555,16 +555,19 @@ void DeltaMergeStore::ingestFiles(
         }
 
         // Check whether all external files are contained by the range.
-        for (const auto & ext_file : external_files)
+        if (dm_context->db_context.getSettingsRef().dt_enable_ingest_check)
         {
-            RUNTIME_CHECK(
-                compare(range.getStart(), ext_file.range.getStart()) <= 0,
-                range.toDebugString(),
-                ext_file.range.toDebugString());
-            RUNTIME_CHECK(
-                compare(range.getEnd(), ext_file.range.getEnd()) >= 0,
-                range.toDebugString(),
-                ext_file.range.toDebugString());
+            for (const auto & ext_file : external_files)
+            {
+                RUNTIME_CHECK_MSG(
+                    compare(range.getStart(), ext_file.range.getStart()) <= 0 && compare(range.getEnd(), ext_file.range.getEnd()) >= 0,
+                    "Detected illegal region boundary: range={} file_range={} . "
+                    "TiFlash will exit to prevent data inconsistency. "
+                    "If you accept data inconsistency and want to continue the service, "
+                    "set profiles.default.dt_enable_ingest_check=false .",
+                    range.toDebugString(),
+                    ext_file.range.toDebugString());
+            }
         }
     }