From 0690f7e2c50a8b4cbafadbf5be16544c0e37d410 Mon Sep 17 00:00:00 2001 From: Benoit Petit Date: Mon, 22 Apr 2024 14:06:40 +0200 Subject: [PATCH 1/5] clean: lowered debug messages level --- src/exporters/json.rs | 3 +-- src/sensors/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/exporters/json.rs b/src/exporters/json.rs index c448cd4f..832d7668 100644 --- a/src/exporters/json.rs +++ b/src/exporters/json.rs @@ -2,8 +2,7 @@ use crate::exporters::*; use crate::sensors::Sensor; use regex::Regex; use serde::{Deserialize, Serialize}; -use std::{ - fs::File, +use std::{ fs::File, io::{BufWriter, Write}, path::{Path, PathBuf}, thread, diff --git a/src/sensors/mod.rs b/src/sensors/mod.rs index 1085f9ea..dc792a34 100644 --- a/src/sensors/mod.rs +++ b/src/sensors/mod.rs @@ -171,7 +171,7 @@ impl Topology { let sysinfo_system = System::new_all(); let sysinfo_cores = sysinfo_system.cpus(); - warn!("Sysinfo sees {}", sysinfo_cores.len()); + debug!("Sysinfo sees {}", sysinfo_cores.len()); #[cfg(target_os = "linux")] let cpuinfo = CpuInfo::new().unwrap(); for (id, c) in (0_u16..).zip(sysinfo_cores.iter()) { @@ -1281,7 +1281,7 @@ impl CPUSocket { )); } } else { - warn!("Not enough records for socket"); + info!("Not enough records for socket"); } None } From a484215fcafbaa1f3d16af9129641ae7b14f55c0 Mon Sep 17 00:00:00 2001 From: Benoit Petit Date: Mon, 22 Apr 2024 14:37:03 +0200 Subject: [PATCH 2/5] fix: reformating json output as a list of reports with enclosure --- src/exporters/json.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/exporters/json.rs b/src/exporters/json.rs index 832d7668..d832663a 100644 --- a/src/exporters/json.rs +++ b/src/exporters/json.rs @@ -2,7 +2,8 @@ use crate::exporters::*; use crate::sensors::Sensor; use regex::Regex; use serde::{Deserialize, Serialize}; -use std::{ fs::File, +use std::{ + fs::File, io::{BufWriter, Write}, path::{Path, PathBuf}, thread, @@ -22,6 +23,7 @@ pub struct JsonExporter { container_regex: Option, monitor_resources: bool, watch_containers: bool, + reports: Vec, } // Note: clap::Args automatically generate Args for the fields of this struct, @@ -171,6 +173,10 @@ impl Exporter for JsonExporter { thread::sleep(self.time_step); } } + + // Serialize the report to json + serde_json::to_writer(&mut self.out_writer, &self.reports) + .expect("report should be serializable to JSON"); } fn kind(&self) -> &str { @@ -210,6 +216,7 @@ impl JsonExporter { None => Box::new(std::io::stdout()), }; let out_writer = BufWriter::new(output); + let reports = vec![]; JsonExporter { metric_generator, time_step, @@ -220,6 +227,7 @@ impl JsonExporter { container_regex, monitor_resources, watch_containers: args.containers, + reports, } } @@ -509,9 +517,7 @@ impl JsonExporter { sockets: all_sockets, }; - // Serialize the report to json - serde_json::to_writer(&mut self.out_writer, &report) - .expect("report should be serializable to JSON"); + self.reports.insert(0, report); } None => { info!("No data yet, didn't write report."); From c96c12e2002949b0fad6276ec18db39c550dd77c Mon Sep 17 00:00:00 2001 From: Benoit Petit Date: Tue, 23 Apr 2024 11:58:34 +0200 Subject: [PATCH 3/5] fix: fixing json syntax in json exporter output --- Cargo.lock | 134 ++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 1 + src/exporters/json.rs | 96 ++++++++++++++++++++++++++---- 3 files changed, 208 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc242584..6a714572 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,6 +130,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bumpalo" version = "3.12.0" @@ -166,6 +172,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.24" @@ -201,7 +213,7 @@ checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex", "once_cell", "strsim", @@ -330,6 +342,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + [[package]] name = "curl" version = "0.4.44" @@ -906,9 +928,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libnghttp2-sys" @@ -1037,6 +1059,18 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -1087,7 +1121,7 @@ version = "0.10.51" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97ea2d98598bf9ada7ea6ee8a30fb74f9156b63bbe495d64ec2b87c269d2dda3" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1214,7 +1248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", - "bitflags", + "bitflags 1.3.2", "cfg-if", "concurrent-queue", "libc", @@ -1238,7 +1272,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "byteorder", "chrono", "flate2", @@ -1268,7 +1302,7 @@ version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1299,7 +1333,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1308,7 +1342,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1376,7 +1410,7 @@ version = "0.36.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0af200a3324fa5bcd922e84e9b55a298ea9f431a489f01961acdebc6e908f25" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -1390,7 +1424,7 @@ version = "0.37.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -1434,6 +1468,7 @@ dependencies = [ "clap", "colored", "core_affinity", + "ctrlc", "docker-sync", "hostname", "hyper", @@ -2061,7 +2096,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9db37ecb5b13762d95468a2fc6009d4b2c62801243223aabd44fca13ad13c8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "widestring", "windows-sys 0.45.0", ] @@ -2112,6 +2147,15 @@ dependencies = [ "windows-targets 0.48.0", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -2142,6 +2186,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2154,6 +2214,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + [[package]] name = "windows_aarch64_msvc" version = "0.27.0" @@ -2172,6 +2238,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + [[package]] name = "windows_i686_gnu" version = "0.27.0" @@ -2190,6 +2262,18 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + [[package]] name = "windows_i686_msvc" version = "0.27.0" @@ -2208,6 +2292,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + [[package]] name = "windows_x86_64_gnu" version = "0.27.0" @@ -2226,6 +2316,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -2238,6 +2334,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + [[package]] name = "windows_x86_64_msvc" version = "0.27.0" @@ -2256,6 +2358,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + [[package]] name = "x86" version = "0.52.0" @@ -2263,7 +2371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2781db97787217ad2a2845c396a5efe286f87467a5810836db6d74926e94a385" dependencies = [ "bit_field", - "bitflags", + "bitflags 1.3.2", "raw-cpuid", ] diff --git a/Cargo.toml b/Cargo.toml index e08a3523..98310423 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ hyper = { version = "0.14", features = ["full"], optional = true } tokio = { version = "1.26.0", features = ["full"], optional = true} sysinfo = { version = "0.28.3"} isahc = { version = "1.7.2", optional = true } +ctrlc = { version = "3.4", features = [ "termination" ] } [target.'cfg(target_os="linux")'.dependencies] procfs = { version = "0.15.0" } diff --git a/src/exporters/json.rs b/src/exporters/json.rs index d832663a..796584dc 100644 --- a/src/exporters/json.rs +++ b/src/exporters/json.rs @@ -2,10 +2,13 @@ use crate::exporters::*; use crate::sensors::Sensor; use regex::Regex; use serde::{Deserialize, Serialize}; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::{ fs::File, io::{BufWriter, Write}, path::{Path, PathBuf}, + sync::Arc, thread, time::{Duration, Instant}, }; @@ -23,7 +26,6 @@ pub struct JsonExporter { container_regex: Option, monitor_resources: bool, watch_containers: bool, - reports: Vec, } // Note: clap::Args automatically generate Args for the fields of this struct, @@ -161,22 +163,68 @@ impl Exporter for JsonExporter { let step = self.time_step; info!("Measurement step is: {step:?}"); + // Serialize the report to json + let res = self.out_writer.write("[".as_bytes()); + + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't write in out_writer: {}", e); + } + } + if let Some(timeout) = self.time_limit { let t0 = Instant::now(); while t0.elapsed() <= timeout { self.iterate(); thread::sleep(self.time_step); + if timeout <= t0.elapsed() { + let res = self.out_writer.write("]".as_bytes()); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't write in out_writer: {}", e); + } + } + } else { + let res = self.out_writer.write(",".as_bytes()); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't write in out_writer: {}", e); + } + } + } } } else { - loop { + let running: Arc = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + ctrlc::set_handler(move || { + println!("received Ctrl+C!"); + r.store(false, Ordering::SeqCst) + }) + .expect("Error setting Ctrl-C handler"); + while running.load(Ordering::SeqCst) { self.iterate(); thread::sleep(self.time_step); + if running.load(Ordering::SeqCst) { + let res = self.out_writer.write(",".as_bytes()); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't write in out_writer: {}", e); + } + } + } + } + let res = self.out_writer.write("]".as_bytes()); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't write in out_writer: {}", e); + } } } - - // Serialize the report to json - serde_json::to_writer(&mut self.out_writer, &self.reports) - .expect("report should be serializable to JSON"); } fn kind(&self) -> &str { @@ -215,8 +263,8 @@ impl JsonExporter { } None => Box::new(std::io::stdout()), }; - let out_writer = BufWriter::new(output); - let reports = vec![]; + let out_writer = + BufWriter::with_capacity(8000 * (time_step.as_secs() as usize / 10), output); JsonExporter { metric_generator, time_step, @@ -227,7 +275,6 @@ impl JsonExporter { container_regex, monitor_resources, watch_containers: args.containers, - reports, } } @@ -517,10 +564,39 @@ impl JsonExporter { sockets: all_sockets, }; - self.reports.insert(0, report); + // Serialize the report to json + serde_json::to_writer(&mut self.out_writer, &report) + .expect("report should be serializable to JSON"); + let res = self.out_writer.flush(); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't flush out_writer: {}", e); + } + } } None => { info!("No data yet, didn't write report."); + let report = Report { + host: Host { + timestamp: current_system_time_since_epoch().as_secs_f64(), + consumption: 0.0, + components: Components { + disks: Some(vec![]), + }, + }, + consumers: vec![], + sockets: vec![], + }; + serde_json::to_writer(&mut self.out_writer, &report) + .expect("report should be serializable to JSON"); + let res = self.out_writer.flush(); + match res { + Ok(_) => {} + Err(e) => { + warn!("Couldn't flush out_writer: {}", e); + } + } } } } From f31dd3e81cc00dbfe111e73bfff42c9c94b1254c Mon Sep 17 00:00:00 2001 From: Benoit Petit Date: Wed, 24 Apr 2024 16:32:03 +0200 Subject: [PATCH 4/5] docs: fixing dead link --- docs_src/references/exporter-json.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_src/references/exporter-json.md b/docs_src/references/exporter-json.md index 30fae878..36d8a103 100644 --- a/docs_src/references/exporter-json.md +++ b/docs_src/references/exporter-json.md @@ -56,4 +56,4 @@ As always exporter's options can be displayed with `-h`: -h, --help Print help -Metrics provided Scaphandre are documented [here](references/metrics.md). \ No newline at end of file +Metrics provided Scaphandre are documented [here](metrics.md). From fc0bc8f5b6b5da5ad50f54b5796d1181686b3c69 Mon Sep 17 00:00:00 2001 From: Benoit Petit Date: Mon, 29 Apr 2024 17:12:31 +0200 Subject: [PATCH 5/5] docs: fixing dead links --- docs_src/references/exporter-prometheus.md | 2 +- docs_src/references/exporter-riemann.md | 2 +- docs_src/references/exporter-warp10.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs_src/references/exporter-prometheus.md b/docs_src/references/exporter-prometheus.md index ee3cd158..ee3d285c 100644 --- a/docs_src/references/exporter-prometheus.md +++ b/docs_src/references/exporter-prometheus.md @@ -34,4 +34,4 @@ With default options values, the metrics are exposed on http://localhost:8080/me Use -q or --qemu option if you are running scaphandre on a hypervisor. In that case a label with the vm name will be added to all `qemu-system*` processes. This will allow to easily create charts consumption for each vm and defined which one is the top contributor. -Metrics provided Scaphandre are documented [here](references/metrics.md). \ No newline at end of file +Metrics provided Scaphandre are documented [here](metrics.md). diff --git a/docs_src/references/exporter-riemann.md b/docs_src/references/exporter-riemann.md index 255089d1..c7c523b8 100644 --- a/docs_src/references/exporter-riemann.md +++ b/docs_src/references/exporter-riemann.md @@ -87,7 +87,7 @@ As a reference here is a Riemann configuration: ## Metrics exposed -Metrics provided Scaphandre are documented [here](references/metrics.md). +Metrics provided Scaphandre are documented [here](metrics.md). There is only one exception about `process_power_consumption_microwatts` each process has a service name `process_power_consumption_microwatts_pid_exe`. diff --git a/docs_src/references/exporter-warp10.md b/docs_src/references/exporter-warp10.md index 92a8be1f..887cfd4b 100644 --- a/docs_src/references/exporter-warp10.md +++ b/docs_src/references/exporter-warp10.md @@ -34,4 +34,4 @@ With default options values, the metrics are sent to http://localhost:8080 every Use -q or --qemu option if you are running scaphandre on a hypervisor. In that case a label with the vm name will be added to all `qemu-system*` processes. This will allow to easily create charts consumption for each vm and defined which one is the top contributor. -Metrics provided Scaphandre are documented [here](references/metrics.md). \ No newline at end of file +Metrics provided Scaphandre are documented [here](metrics.md).