|
| 1 | +// This file is Copyright its original authors, visible in version control history. |
| 2 | +// |
| 3 | +// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| 4 | +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or |
| 5 | +// http://opensource.org/licenses/MIT>, at your option. You may not use this file except in |
| 6 | +// accordance with one or both of these licenses. |
| 7 | + |
| 8 | +use std::sync::atomic::{AtomicBool, Ordering}; |
| 9 | +use std::sync::{Arc, Mutex}; |
| 10 | +use std::time::{Duration, Instant}; |
| 11 | + |
| 12 | +use lightning::io; |
| 13 | + |
| 14 | +pub(crate) const NODE_LEASE_DURATION: Duration = Duration::from_secs(30); |
| 15 | +// Fail closed before the database lease expires, leaving time for process termination. |
| 16 | +pub(crate) const NODE_LEASE_RENEWAL_DEADLINE: Duration = Duration::from_secs(20); |
| 17 | +pub(crate) const NODE_LEASE_RENEWAL_INTERVAL: Duration = Duration::from_secs(10); |
| 18 | +pub(crate) const NODE_LEASE_RETRY_INTERVAL: Duration = Duration::from_secs(1); |
| 19 | +pub(crate) const NODE_LEASE_RELEASE_TIMEOUT: Duration = Duration::from_secs(5); |
| 20 | + |
| 21 | +type LeaseLossHandler = Box<dyn FnOnce() + Send>; |
| 22 | + |
| 23 | +pub(crate) struct NodeLease { |
| 24 | + owner_id: [u8; 32], |
| 25 | + lease_lost: AtomicBool, |
| 26 | + last_confirmed_renewal: Mutex<Instant>, |
| 27 | + loss_sender: tokio::sync::watch::Sender<bool>, |
| 28 | + loss_handler: Mutex<Option<LeaseLossHandler>>, |
| 29 | +} |
| 30 | + |
| 31 | +impl NodeLease { |
| 32 | + pub(crate) fn new() -> io::Result<Arc<Self>> { |
| 33 | + let mut owner_id = [0u8; 32]; |
| 34 | + getrandom::fill(&mut owner_id).map_err(|e| { |
| 35 | + io::Error::new(io::ErrorKind::Other, format!("Failed to generate lease owner ID: {e}")) |
| 36 | + })?; |
| 37 | + let (loss_sender, _) = tokio::sync::watch::channel(false); |
| 38 | + Ok(Arc::new(Self { |
| 39 | + owner_id, |
| 40 | + lease_lost: AtomicBool::new(false), |
| 41 | + last_confirmed_renewal: Mutex::new(Instant::now()), |
| 42 | + loss_sender, |
| 43 | + loss_handler: Mutex::new(None), |
| 44 | + })) |
| 45 | + } |
| 46 | + |
| 47 | + pub(crate) fn owner_id(&self) -> &[u8; 32] { |
| 48 | + &self.owner_id |
| 49 | + } |
| 50 | + |
| 51 | + pub(crate) fn is_lost(&self) -> bool { |
| 52 | + self.lease_lost.load(Ordering::Acquire) |
| 53 | + } |
| 54 | + |
| 55 | + pub(crate) fn record_renewal_started_at(&self, renewal_started_at: Instant) { |
| 56 | + if !self.is_lost() { |
| 57 | + let mut last_confirmed_renewal = self.last_confirmed_renewal.lock().expect("lock"); |
| 58 | + *last_confirmed_renewal = (*last_confirmed_renewal).max(renewal_started_at); |
| 59 | + } |
| 60 | + } |
| 61 | + |
| 62 | + pub(crate) fn renewal_deadline_elapsed(&self) -> bool { |
| 63 | + self.last_confirmed_renewal.lock().expect("lock").elapsed() >= NODE_LEASE_RENEWAL_DEADLINE |
| 64 | + } |
| 65 | + |
| 66 | + pub(crate) async fn wait_for_renewal_deadline(&self) { |
| 67 | + loop { |
| 68 | + let last_confirmed_renewal = *self.last_confirmed_renewal.lock().expect("lock"); |
| 69 | + let deadline = last_confirmed_renewal + NODE_LEASE_RENEWAL_DEADLINE; |
| 70 | + tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)).await; |
| 71 | + if self.renewal_deadline_elapsed() { |
| 72 | + return; |
| 73 | + } |
| 74 | + } |
| 75 | + } |
| 76 | + |
| 77 | + pub(crate) fn ensure_operation_active(&self) -> io::Result<()> { |
| 78 | + if self.is_lost() || self.renewal_deadline_elapsed() { |
| 79 | + self.mark_lost(); |
| 80 | + Err(lease_lost_error()) |
| 81 | + } else { |
| 82 | + Ok(()) |
| 83 | + } |
| 84 | + } |
| 85 | + |
| 86 | + pub(crate) fn map_operation_error(&self, error: io::Error) -> io::Error { |
| 87 | + // Preserve transient database errors until they outlive the local safety margin. |
| 88 | + self.ensure_operation_active().err().unwrap_or(error) |
| 89 | + } |
| 90 | + |
| 91 | + pub(crate) fn mark_lost(&self) { |
| 92 | + if self |
| 93 | + .lease_lost |
| 94 | + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) |
| 95 | + .is_err() |
| 96 | + { |
| 97 | + return; |
| 98 | + } |
| 99 | + |
| 100 | + // Run any installed containment handler before publishing lease loss. |
| 101 | + if let Some(handler) = self.loss_handler.lock().expect("lock").take() { |
| 102 | + handler(); |
| 103 | + } |
| 104 | + self.loss_sender.send_replace(true); |
| 105 | + } |
| 106 | + |
| 107 | + pub(crate) fn set_loss_handler(&self, handler: LeaseLossHandler) { |
| 108 | + let mut locked_handler = self.loss_handler.lock().expect("lock"); |
| 109 | + if self.is_lost() { |
| 110 | + drop(locked_handler); |
| 111 | + handler(); |
| 112 | + } else { |
| 113 | + *locked_handler = Some(handler); |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + pub(crate) async fn wait_for_loss(self: Arc<Self>) { |
| 118 | + let mut receiver = self.loss_sender.subscribe(); |
| 119 | + let _ = receiver.wait_for(|lost| *lost).await; |
| 120 | + } |
| 121 | +} |
| 122 | + |
| 123 | +pub(crate) fn lease_lost_error() -> io::Error { |
| 124 | + io::Error::new(io::ErrorKind::PermissionDenied, "PostgreSQL node lease was lost") |
| 125 | +} |
| 126 | + |
| 127 | +#[cfg(test)] |
| 128 | +mod tests { |
| 129 | + use std::sync::atomic::{AtomicBool, Ordering}; |
| 130 | + |
| 131 | + use super::*; |
| 132 | + |
| 133 | + #[test] |
| 134 | + fn expired_operation_marks_loss_before_returning_error() { |
| 135 | + let lease = NodeLease::new().unwrap(); |
| 136 | + let handler_ran = Arc::new(AtomicBool::new(false)); |
| 137 | + let handler_ran_ref = Arc::clone(&handler_ran); |
| 138 | + lease.set_loss_handler(Box::new(move || { |
| 139 | + handler_ran_ref.store(true, Ordering::Release); |
| 140 | + })); |
| 141 | + *lease.last_confirmed_renewal.lock().unwrap() = |
| 142 | + Instant::now() - NODE_LEASE_RENEWAL_DEADLINE; |
| 143 | + |
| 144 | + let error = lease.map_operation_error(io::Error::from(io::ErrorKind::Other)); |
| 145 | + |
| 146 | + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); |
| 147 | + assert!(lease.is_lost()); |
| 148 | + assert!(handler_ran.load(Ordering::Acquire)); |
| 149 | + } |
| 150 | + |
| 151 | + #[test] |
| 152 | + fn confirmed_renewal_uses_attempt_time_and_does_not_regress() { |
| 153 | + let lease = NodeLease::new().unwrap(); |
| 154 | + let renewal_started_at = Instant::now() - Duration::from_secs(1); |
| 155 | + *lease.last_confirmed_renewal.lock().unwrap() = renewal_started_at - Duration::from_secs(1); |
| 156 | + |
| 157 | + lease.record_renewal_started_at(renewal_started_at); |
| 158 | + lease.record_renewal_started_at(renewal_started_at - Duration::from_secs(1)); |
| 159 | + |
| 160 | + assert_eq!(*lease.last_confirmed_renewal.lock().unwrap(), renewal_started_at); |
| 161 | + } |
| 162 | + |
| 163 | + #[tokio::test] |
| 164 | + async fn expired_renewal_deadline_completes_immediately() { |
| 165 | + let lease = NodeLease::new().unwrap(); |
| 166 | + *lease.last_confirmed_renewal.lock().unwrap() = |
| 167 | + Instant::now() - NODE_LEASE_RENEWAL_DEADLINE; |
| 168 | + |
| 169 | + tokio::time::timeout(Duration::from_secs(1), lease.wait_for_renewal_deadline()) |
| 170 | + .await |
| 171 | + .unwrap(); |
| 172 | + } |
| 173 | +} |
0 commit comments