From 12bdfff675ce9c4854844c3e8419e686fb984081 Mon Sep 17 00:00:00 2001
From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com>
Date: Mon, 18 Nov 2024 22:27:15 -0500
Subject: [PATCH] `refactor`: get_stats_records helper to ignore dataset-level
 stats and use simd_json instead of serde_json

---
 src/util.rs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/util.rs b/src/util.rs
index 35bf70d8f..436ead86b 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -1968,10 +1968,17 @@ pub fn get_stats_records(
         let statsdata_lines = statsdata_reader.lines();
 
         let mut line: String;
+        let mut s_slice: Vec<u8>;
         for curr_line in statsdata_lines {
             line = curr_line?;
-            let stats_record: StatsData = serde_json::from_str(&line)?;
-            csv_stats.push(stats_record);
+            // do not load dataset-level stats into csv_stats
+            if !line.starts_with(r#"{"field":"_qsv_"#) {
+                s_slice = line.as_bytes().to_vec();
+                match simd_json::serde::from_slice(&mut **&mut s_slice) {
+                    Ok(stats) => csv_stats.push(stats),
+                    Err(_) => continue,
+                }
+            }
         }
         stats_data_loaded = true;
     }