diff --git a/.gitignore b/.gitignore index 9f44766f..1b76b4a1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,4 @@ test-ledger/ # Allow specific json files !tests/fixtures/**/*.json -!programs/marginfi/tests/fixtures/**/*.json +!programs/marginfi/tests/fixtures/**/*.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0eaf5add..5d2242bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "alerting" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytemuck", + "chrono", + "env_logger 0.11.5", + "log", + "marginfi", + "pagerduty-rs", + "pyth-sdk-solana", + "pyth-solana-receiver-sdk", + "serde", + "solana-account-decoder", + "solana-client", + "solana-sdk", + "structopt", + "switchboard-on-demand", + "switchboard-solana", + "time", + "toml 0.8.19", +] + [[package]] name = "aliasable" version = "0.1.3" @@ -502,11 +526,60 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "aquamarine" @@ -1137,9 +1210,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] @@ -1205,7 +1278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" dependencies = [ "serde", - "toml 0.8.14", + "toml 0.8.19", ] [[package]] @@ -1318,6 +1391,12 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + [[package]] name = "combine" version = "3.8.1" @@ -1872,6 +1951,16 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -1885,6 +1974,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -2223,7 +2325,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util 0.7.11", @@ -2282,6 +2384,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" version = "0.3.3" @@ -2546,12 +2654,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -2582,6 +2690,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2814,7 +2928,7 @@ dependencies = [ "anyhow", "bytemuck", "clap 3.2.25", - "env_logger", + "env_logger 0.9.3", "fixed", "fixed-macro", "futures", @@ -2935,7 +3049,7 @@ dependencies = [ "chrono", "clap 3.2.25", "dirs", - "env_logger", + "env_logger 0.9.3", "fixed", "fixed-macro", "hex", @@ -3534,6 +3648,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "pagerduty-rs" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd10bab2b6df910bbe6c4987d76aa4221235103d9a9c000cfabcee6a6abc8f7a" +dependencies = [ + "reqwest", + "serde", + "time", + "url", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -3850,9 +3976,9 @@ dependencies = [ [[package]] name = "pyth-solana-receiver-sdk" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e6559643f0b377b6f293269251f6a804ae7332c37f7310371f50c833453cd0" +checksum = "1b7854c4176470c8d86de301dc5b57ac84227dabb9527328b585fc332962d60b" dependencies = [ "anchor-lang 0.29.0", "hex", @@ -4579,9 +4705,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -4597,9 +4723,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -4632,7 +4758,7 @@ version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -4640,9 +4766,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -4687,7 +4813,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -5120,7 +5246,7 @@ dependencies = [ "dashmap", "futures", "futures-util", - "indexmap 2.2.6", + "indexmap 2.6.0", "indicatif", "log", "quinn", @@ -5176,7 +5302,7 @@ dependencies = [ "bincode", "crossbeam-channel", "futures-util", - "indexmap 2.2.6", + "indexmap 2.6.0", "log", "rand 0.8.5", "rayon", @@ -5268,7 +5394,7 @@ version = "1.18.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0511082fc62f2d086520fff5aa1917c389d8c840930c08ad255ae05952c08a2" dependencies = [ - "env_logger", + "env_logger 0.9.3", "lazy_static", "log", ] @@ -5803,7 +5929,7 @@ dependencies = [ "crossbeam-channel", "futures-util", "histogram", - "indexmap 2.2.6", + "indexmap 2.6.0", "itertools", "libc", "log", @@ -5863,7 +5989,7 @@ dependencies = [ "async-trait", "bincode", "futures-util", - "indexmap 2.2.6", + "indexmap 2.6.0", "indicatif", "log", "rayon", @@ -6540,6 +6666,30 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "strum" version = "0.24.1" @@ -7196,21 +7346,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.15", + "toml_edit 0.22.22", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -7221,7 +7371,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "toml_datetime", "winnow 0.5.40", ] @@ -7232,22 +7382,22 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.13", + "winnow 0.6.20", ] [[package]] @@ -7476,6 +7626,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.10.0" @@ -7873,9 +8029,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/tools/alerting/Cargo.toml b/tools/alerting/Cargo.toml new file mode 100644 index 00000000..44794903 --- /dev/null +++ b/tools/alerting/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "alerting" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.89" +bytemuck = "1.19.0" +chrono = "0.4.38" +env_logger = "0.11.5" +log = "0.4.22" +marginfi = { path = "../../programs/marginfi", version = "0.1.0", features = [ + "mainnet-beta", + "client", + "no-entrypoint", +] } +pagerduty-rs = { version = "*", features = ["sync"] } +pyth-sdk-solana = { workspace = true } +pyth-solana-receiver-sdk = "0.3.1" +serde = "1.0.210" +solana-account-decoder = { workspace = true } +solana-client.workspace = true +solana-sdk.workspace = true +structopt = "0.3.26" +switchboard-on-demand ={ workspace = true } +switchboard-solana ={ workspace = true } +time = "0.3.36" +toml = "0.8.19" diff --git a/tools/alerting/src/main.rs b/tools/alerting/src/main.rs new file mode 100644 index 00000000..4c2f130d --- /dev/null +++ b/tools/alerting/src/main.rs @@ -0,0 +1,514 @@ +use std::{ + collections::HashMap, + mem::size_of, + str::FromStr, + time::{SystemTime, UNIX_EPOCH}, +}; + +use log::{error, info, warn}; +use marginfi::{ + constants::{PYTH_PUSH_MARGINFI_SPONSORED_SHARD_ID, PYTH_PUSH_PYTH_SPONSORED_SHARD_ID}, + state::{marginfi_group::Bank, price::OracleSetup}, +}; +use pagerduty_rs::{ + eventsv2sync::EventsV2, + types::{AlertResolve, AlertTrigger, AlertTriggerPayload, Event}, +}; +use pyth_solana_receiver_sdk::price_update::PriceUpdateV2; +use serde::Deserialize; +use solana_account_decoder::UiAccountEncoding; +use solana_client::{ + rpc_client::RpcClient, + rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}, + rpc_filter::{Memcmp, RpcFilterType}, +}; +use solana_sdk::pubkey::Pubkey; +use structopt::StructOpt; +use switchboard_on_demand::PullFeedAccountData; +use switchboard_solana::{AggregatorAccountData, AnchorDeserialize}; +use time::OffsetDateTime; + +#[derive(Clone, Debug, Deserialize)] +pub struct MarginfiAlerterConfig { + rpc_url: String, + pd_integration_key: String, + marginfi_program_id: String, + marginfi_groups: Vec, + balance_alert: Vec, +} + +impl MarginfiAlerterConfig { + pub fn load_from_file(path: &str) -> Result> { + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + let config_str = std::io::read_to_string(reader)?; + let config = toml::from_str(&config_str)?; + Ok(config) + } +} + +#[derive(Clone, Debug, Deserialize)] +struct MarginfiGroupAlertingConfig { + address: String, + max_age_secs: i64, +} + +#[derive(Clone, Debug, Deserialize)] +struct BalanceAlertingConfig { + address: String, + min_balance: f64, + label: Option, +} + +#[derive(Clone, Debug, StructOpt)] +pub struct MarginfiAlerter { + config_path: String, +} + +struct AlertingContext { + rpc_client: RpcClient, + config: MarginfiAlerterConfig, + group_config_map: HashMap, + pd: EventsV2, +} + +fn main() { + env_logger::init(); + let args = MarginfiAlerter::from_args(); + let config = MarginfiAlerterConfig::load_from_file(&args.config_path).unwrap(); + let rpc_client = RpcClient::new(config.rpc_url.clone()); + + println!( + "Starting marginfi alerter, evaluating {} groups and {} balance alerts", + config.marginfi_groups.len(), + config.balance_alert.len() + ); + + let group_config_map = config + .marginfi_groups + .iter() + .map(|group| (Pubkey::from_str(&group.address).unwrap(), group.clone())) + .collect(); + + let context = AlertingContext { + rpc_client, + config: config.clone(), + group_config_map, + pd: EventsV2::new(config.pd_integration_key.clone(), None).unwrap(), + }; + + match check_all(&context) { + Ok(_) => { + clear_error_alert(&context).unwrap(); + } + Err(e) => { + error!("Error running marginfi alerter: {:?}", e); + send_error_alert(&context, &format!("Error running marginfi alerter: {}", e)).unwrap(); + eprintln!("{:#?}", e); + std::process::exit(1); + } + } +} + +fn check_all(context: &AlertingContext) -> anyhow::Result<()> { + check_marginfi_groups(context)?; + check_balance_alert(context)?; + Ok(()) +} + +fn send_error_alert(context: &AlertingContext, message: &str) -> anyhow::Result<()> { + let event = Event::AlertTrigger::(AlertTrigger { + payload: AlertTriggerPayload { + summary: message.to_string(), + source: "marginfi-alerter".to_string(), + severity: pagerduty_rs::types::Severity::Critical, + custom_details: None, + component: None, + group: None, + class: None, + timestamp: None, + }, + dedup_key: Some("marginfi-alerter-error".to_string()), + images: None, + links: None, + client: None, + client_url: None, + }); + + context.pd.event(event).map_err(|e| anyhow::anyhow!(e))?; + + Ok(()) +} + +fn clear_error_alert(context: &AlertingContext) -> anyhow::Result<()> { + let event = Event::AlertResolve::(AlertResolve { + dedup_key: "marginfi-alerter-error".to_string(), + }); + + context.pd.event(event).map_err(|e| anyhow::anyhow!(e))?; + + Ok(()) +} + +fn check_marginfi_groups(context: &AlertingContext) -> anyhow::Result<()> { + for group in context.config.marginfi_groups.iter() { + check_marginfi_group(context, group)?; + } + + Ok(()) +} + +fn check_balance_alert(context: &AlertingContext) -> anyhow::Result<()> { + for balance in context.config.balance_alert.iter() { + check_balance(context, balance)?; + } + + Ok(()) +} + +fn check_balance( + context: &AlertingContext, + balance_config: &BalanceAlertingConfig, +) -> anyhow::Result<()> { + let address = Pubkey::from_str(&balance_config.address)?; + let balance = context.rpc_client.get_balance(&address)?; + + let min_balance_lamports = (balance_config.min_balance * 10u64.pow(9) as f64).floor() as u64; + + info!( + "Balance for account {} is {}, min balance is {}", + address, + balance as f64 / 10u64.pow(9) as f64, + balance_config.min_balance + ); + + if balance < min_balance_lamports { + send_balance_alert(context, balance_config, balance)?; + } else { + clear_balance_alert(context, &address)?; + } + + Ok(()) +} + +fn check_marginfi_group( + context: &AlertingContext, + group: &MarginfiGroupAlertingConfig, +) -> anyhow::Result<()> { + let marginfi_program_id = Pubkey::from_str(&context.config.marginfi_program_id)?; + let group_address = Pubkey::from_str(&group.address)?; + let bank_accounts = context.rpc_client.get_program_accounts_with_config( + &marginfi_program_id, + RpcProgramAccountsConfig { + filters: Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes( + 8 + size_of::() + size_of::(), + group_address.to_bytes().to_vec(), + ))]), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + ..Default::default() + }, + with_context: None, + }, + )?; + + let banks = bank_accounts + .into_iter() + .map(|(address, account)| { + let data = account.data.as_slice(); + bytemuck::try_from_bytes::(&data[8..]) + .cloned() + .map(|b| (address, b)) + }) + .collect::, _>>() + .map_err(|e| anyhow::anyhow!(e))?; + + info!("Found {} banks in group", banks.len()); + + let switchboard_v2_oracles = banks + .iter() + .filter(|(_, bank)| bank.config.oracle_setup == OracleSetup::SwitchboardV2) + .collect::>(); + // Pyth legacy is deprecated + let _pyth_oracles = banks + .iter() + .filter(|(_, bank)| bank.config.oracle_setup == OracleSetup::PythLegacy) + .collect::>(); + let pyth_push_oracles = banks + .iter() + .filter(|(_, bank)| bank.config.oracle_setup == OracleSetup::PythPushOracle) + .collect::>(); + let switchboard_pull_oracles = banks + .iter() + .filter(|(_, bank)| bank.config.oracle_setup == OracleSetup::SwitchboardPull) + .collect::>(); + + check_switchboard_v2_oracles(context, &switchboard_v2_oracles)?; + check_pyth_push_oracles(context, &pyth_push_oracles)?; + check_switchboard_pull_oracles(context, &switchboard_pull_oracles)?; + + Ok(()) +} + +fn check_switchboard_v2_oracles( + context: &AlertingContext, + banks: &[&(Pubkey, Bank)], +) -> anyhow::Result<()> { + info!("Checking {} switchboard v2 oracles", banks.len()); + for (address, bank) in banks { + check_switchboard_v2_oracle(context, address, bank)?; + } + + Ok(()) +} + +fn check_switchboard_v2_oracle( + context: &AlertingContext, + address: &Pubkey, + bank: &Bank, +) -> anyhow::Result<()> { + let oracle_address = bank.config.oracle_keys.first().unwrap(); + let oracle_account = context.rpc_client.get_account(oracle_address)?; + let oracle = bytemuck::try_from_bytes::(&oracle_account.data[8..]) + .map_err(|e| anyhow::anyhow!(e))?; + let group_config = context.group_config_map.get(&bank.group).unwrap(); + let max_age = group_config.max_age_secs; + let last_update = oracle.latest_confirmed_round.round_open_timestamp; + let current_time = get_current_unix_timestamp_secs(); + let oracle_age = current_time - last_update; + + info!( + "Switchboard V2 oracle for bank {} is {} seconds old", + address, oracle_age + ); + + if oracle_age > max_age { + send_stale_oracle_alert(context, address, oracle_age)?; + } else { + clear_stale_oracle_alert(context, address)?; + } + + Ok(()) +} + +fn check_pyth_push_oracles( + context: &AlertingContext, + banks: &[&(Pubkey, Bank)], +) -> anyhow::Result<()> { + for (address, bank) in banks { + check_pyth_push_oracle(context, address, bank)?; + } + + Ok(()) +} + +fn check_pyth_push_oracle( + context: &AlertingContext, + address: &Pubkey, + bank: &Bank, +) -> anyhow::Result<()> { + let oracle_address = bank.config.oracle_keys.first().unwrap(); + let oracle_account = { + let (marginfi_sponsored_oracle_address, _) = + marginfi::state::price::PythPushOraclePriceFeed::find_oracle_address( + PYTH_PUSH_MARGINFI_SPONSORED_SHARD_ID, + oracle_address.as_ref().try_into().unwrap(), + ); + let (pyth_sponsered_oracle_address, _) = + marginfi::state::price::PythPushOraclePriceFeed::find_oracle_address( + PYTH_PUSH_PYTH_SPONSORED_SHARD_ID, + oracle_address.as_ref().try_into().unwrap(), + ); + + let accounts = context.rpc_client.get_multiple_accounts(&[ + marginfi_sponsored_oracle_address, + pyth_sponsered_oracle_address, + ])?; + + match (accounts.first().cloned(), accounts.get(1).cloned()) { + (Some(Some(account)), _) => account, + (_, Some(Some(account))) => account, + _ => anyhow::bail!("Oracle account for bank {} not found", address), + } + }; + + let price_update = PriceUpdateV2::deserialize(&mut &oracle_account.data[8..])?; + let group_config = context.group_config_map.get(&bank.group).unwrap(); + let publish_time = price_update.price_message.publish_time; + let current_time = get_current_unix_timestamp_secs(); + let max_age = group_config.max_age_secs; + let oracle_age = current_time - publish_time; + + info!( + "Pyth push oracle for bank {} is {} seconds old", + address, oracle_age + ); + + if oracle_age > max_age { + send_stale_oracle_alert(context, address, oracle_age)?; + } else { + clear_stale_oracle_alert(context, address)?; + } + + Ok(()) +} + +fn check_switchboard_pull_oracles( + context: &AlertingContext, + banks: &[&(Pubkey, Bank)], +) -> anyhow::Result<()> { + for (address, bank) in banks { + check_switchboard_pull_oracle(context, address, bank)?; + } + + Ok(()) +} + +fn check_switchboard_pull_oracle( + context: &AlertingContext, + address: &Pubkey, + bank: &Bank, +) -> anyhow::Result<()> { + let oracle_address = bank.config.oracle_keys.first().unwrap(); + let oracle_account = context.rpc_client.get_account(oracle_address)?; + let pull_feed = bytemuck::try_from_bytes::(&oracle_account.data[8..]) + .map_err(|e| anyhow::anyhow!(e))?; + let group_config = context.group_config_map.get(&bank.group).unwrap(); + let max_age = group_config.max_age_secs; + let current_time = get_current_unix_timestamp_secs(); + let last_update = pull_feed.last_update_timestamp; + let oracle_age = current_time - last_update; + + info!( + "Switchboard pull oracle for bank {} is {} seconds old", + address, oracle_age + ); + + if oracle_age > max_age { + send_stale_oracle_alert(context, address, oracle_age)?; + } else { + clear_stale_oracle_alert(context, address)?; + } + + Ok(()) +} + +fn send_balance_alert( + context: &AlertingContext, + balance_config: &BalanceAlertingConfig, + balance: u64, +) -> anyhow::Result<()> { + let balance_ui = balance as f64 / 10u64.pow(9) as f64; + warn!( + "Account {} ({}) has balance of {} below minimum {}", + balance_config.address, + balance_config + .label + .clone() + .unwrap_or(balance_config.address.to_string()), + balance_ui, + balance_config.min_balance + ); + + context + .pd + .event(pagerduty_rs::types::Event::AlertTrigger::( + AlertTrigger { + payload: AlertTriggerPayload { + severity: pagerduty_rs::types::Severity::Critical, + summary: format!( + "Account {} ({}) has balance of {} below minimum {}", + balance_config.address, + balance_config + .label + .clone() + .unwrap_or(balance_config.address.to_string()), + balance_ui, + balance_config.min_balance + ), + source: "marginfi-alerter".to_string(), + timestamp: Some(OffsetDateTime::now_utc()), + component: None, + group: None, + class: None, + custom_details: None, + }, + dedup_key: Some(format!("balance-{}", balance_config.address)), + images: None, + links: None, + client: None, + client_url: None, + }, + ))?; + + Ok(()) +} + +fn clear_balance_alert(context: &AlertingContext, address: &Pubkey) -> anyhow::Result<()> { + context + .pd + .event(Event::AlertResolve::(AlertResolve { + dedup_key: format!("balance-{}", address), + }))?; + + Ok(()) +} + +fn send_stale_oracle_alert( + context: &AlertingContext, + address: &Pubkey, + oralce_age_secs: i64, +) -> anyhow::Result<()> { + warn!( + "Oracle for bank {} is stale by {} seconds, sending alert", + address, oralce_age_secs + ); + + context + .pd + .event(pagerduty_rs::types::Event::AlertTrigger::( + AlertTrigger { + payload: AlertTriggerPayload { + severity: pagerduty_rs::types::Severity::Critical, + summary: format!( + "Oracle for bank {} is stale by {} seconds", + address, oralce_age_secs + ), + source: "marginfi-alerter".to_string(), + timestamp: Some(OffsetDateTime::now_utc()), + component: None, + group: None, + class: None, + custom_details: None, + }, + dedup_key: Some(get_oracle_dedup_key(address)), + images: None, + links: None, + client: None, + client_url: None, + }, + ))?; + + Ok(()) +} + +fn clear_stale_oracle_alert(context: &AlertingContext, address: &Pubkey) -> anyhow::Result<()> { + context + .pd + .event(Event::AlertResolve::(AlertResolve { + dedup_key: get_oracle_dedup_key(address), + }))?; + + Ok(()) +} + +fn get_oracle_dedup_key(address: &Pubkey) -> String { + format!("stale-oracle-{}", address) +} + +fn get_current_unix_timestamp_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() as i64 +}