feat: add procfs process tracking

Author: preinlein

Diff for: CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Lading now built with edition 2024
 - Removed use of compromised `tj-actions/changed-files` action from project's GitHub CI configuration
 - Fixed devcontainer configuration to ensure the `rust-analyzer` can run successfully within IDEs
+- Added a new gauge `processes_found` and a new warning log for processes we skipped
 
 ## [0.25.6]
 ## Fixed

Diff for: lading/src/observer/linux/procfs.rs

@@ -83,6 +83,8 @@ impl Sampler {
         // A tally of the total RSS and PSS consumed by the parent process and
         // its children.
         let mut aggr = memory::smaps_rollup::Aggregator::default();
+        let mut processes_found: i32 = 0;
+        let mut pids_skipped: FxHashSet<i32> = FxHashSet::default();
 
         // Every sample run we collect all the child processes rooted at the
         // parent. As noted by the procfs documentation is this done by
@@ -115,25 +117,43 @@ impl Sampler {
                                 pids.insert(pid);
                             }
                         }
-                    }
+                    } 
                 }
             }
 
+            processes_found += 1;
             let pid = process.pid();
-            if let Err(e) = self.handle_process(process, &mut aggr, include_smaps).await {
-                warn!("Encountered uncaught error when handling `/proc/{pid}/`: {e}");
-            }
+            match self.handle_process(process, &mut aggr, include_smaps).await {
+                Ok(true) => { /* Handled successfully */ }
+                Ok(false) => {
+                    pids_skipped.insert(pid);
+                }
+                Err(e) => {
+                    warn!("Encountered uncaught error when handling `/proc/{pid}/`: {e}");
+                }
         }
 
         // Update the process_info map to only hold processes seen by the current poll call.
         self.process_info.retain(|pid, _| pids.contains(pid));
 
         gauge!("total_rss_bytes").set(aggr.rss as f64);
         gauge!("total_pss_bytes").set(aggr.pss as f64);
+        gauge!("processes_found").set(processes_found as f64);
+
+        // If we skipped any processes, log a warning.
+        if !pids_skipped.is_empty() {
+            warn!(
+                "Skipped {} processes: {:?}",
+                pids_skipped.len(),
+                pids_skipped
+            );
+        }
 
-        Ok(())
+        ()
     }
 
+    /// Handle a process. Returns true if the process was handled successfully,
+    /// false if it was skipped for any reason.    
     #[allow(
         clippy::similar_names,
         clippy::too_many_lines,
@@ -146,7 +166,7 @@ impl Sampler {
         process: Process,
         aggr: &mut memory::smaps_rollup::Aggregator,
         include_smaps: bool,
-    ) -> Result<(), Error> {
+    ) -> Result<bool, Error> {
         let pid = process.pid();
 
         // `/proc/{pid}/status`
@@ -156,12 +176,12 @@ impl Sampler {
                 warn!("Couldn't read status: {:?}", e);
                 // The pid may have exited since we scanned it or we may not
                 // have sufficient permission.
-                return Ok(());
+                return Ok(false);
             }
         };
         if status.tgid != pid {
             // This is a thread, not a process and we do not wish to scan it.
-            return Ok(());
+            return Ok(false);
         }
 
         // If we haven't seen this process before, initialize its ProcessInfo.
@@ -174,7 +194,7 @@ impl Sampler {
                         warn!("Couldn't read exe for pid {}: {:?}", pid, e);
                         // The pid may have exited since we scanned it or we may not
                         // have sufficient permission.
-                        return Ok(());
+                        return Ok(false);
                     }
                 };
                 let comm = match proc_comm(pid).await {
@@ -183,7 +203,7 @@ impl Sampler {
                         warn!("Couldn't read comm for pid {}: {:?}", pid, e);
                         // The pid may have exited since we scanned it or we may not
                         // have sufficient permission.
-                        return Ok(());
+                        return Ok(false);
                     }
                 };
                 let cmdline = match proc_cmdline(pid).await {
@@ -192,7 +212,7 @@ impl Sampler {
                         warn!("Couldn't read cmdline for pid {}: {:?}", pid, e);
                         // The pid may have exited since we scanned it or we may not
                         // have sufficient permission.
-                        return Ok(());
+                        return Ok(false);
                     }
                 };
                 let pid_s = format!("{pid}");
@@ -238,7 +258,7 @@ impl Sampler {
             // which will happen if we don't have permissions or, more
             // likely, the process has exited.
             warn!("Couldn't process `/proc/{pid}/stat`: {e}");
-            return Ok(());
+            return Ok(false);
         }
 
         if include_smaps {
@@ -317,10 +337,10 @@ impl Sampler {
             // which will happen if we don't have permissions or, more
             // likely, the process has exited.
             warn!("Couldn't process `/proc/{pid}/smaps_rollup`: {err}");
-            return Ok(());
+            return Ok(false);
         }
 
-        Ok(())
+        Ok(true)
     }
 }