Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ pub mod legacy;
pub mod messaging;
pub mod protocol;
pub mod registry;
pub mod shutdown_grace;

pub const VERSION: &str = env!("CARGO_PKG_VERSION");
pub const BUILD_PROFILE: &str = if cfg!(debug_assertions) {
Expand Down
212 changes: 212 additions & 0 deletions backend/src/shutdown_grace.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
//! Graceful shutdown grace-period configuration.
//!
//! The backend supports a configurable shutdown grace period via the
//! `TOT_SHUTDOWN_GRACE_SECS` environment variable. This module parses,
//! validates, and applies bounds to that setting so deploys use it
//! consistently.

use std::env;
use std::time::Duration;

/// Environment variable name for the shutdown grace period.
pub const ENV_SHUTDOWN_GRACE_SECS: &str = "TOT_SHUTDOWN_GRACE_SECS";

/// Default grace period in seconds when the variable is unset.
pub const DEFAULT_GRACE_SECS: u64 = 30;

/// Maximum allowed grace period in seconds (5 minutes).
pub const MAX_GRACE_SECS: u64 = 300;

/// Minimum allowed grace period in seconds.
pub const MIN_GRACE_SECS: u64 = 1;

/// Parsed grace-period result.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GracePeriod {
/// The resolved duration.
pub duration: Duration,
/// How the value was determined.
pub source: GraceSource,
}

/// How the grace period was determined.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GraceSource {
/// No env var set; the default was used.
Default,
/// The env var contained a valid value within bounds.
Env(u64),
/// The env var was zero or below minimum; clamped to the minimum.
ClampedLow { raw: i64, clamped_to: u64 },
/// The env var exceeded the maximum; clamped to the maximum.
ClampedHigh { raw: u64, clamped_to: u64 },
/// The env var was not a valid integer; the default was used instead.
Invalid { raw: String, fallback: u64 },
}

/// Parse the `TOT_SHUTDOWN_GRACE_SECS` environment variable and return a
/// validated [`GracePeriod`].
///
/// | Input | Behaviour |
/// |-----------------------------|---------------------------------------------|
/// | Unset | Use `DEFAULT_GRACE_SECS` |
/// | Valid integer in range | Use the parsed value |
/// | Zero | Clamp to `MIN_GRACE_SECS` |
/// | Negative | Clamp to `MIN_GRACE_SECS` |
/// | Above `MAX_GRACE_SECS` | Clamp to `MAX_GRACE_SECS` |
/// | Non-numeric | Fall back to `DEFAULT_GRACE_SECS` |
pub fn parse_grace_period() -> GracePeriod {
parse_grace_period_from(env::var(ENV_SHUTDOWN_GRACE_SECS).ok())
}

/// Testable entry-point that accepts an optional raw string instead of
/// reading the real environment.
pub fn parse_grace_period_from(raw: Option<String>) -> GracePeriod {
match raw {
None => GracePeriod {
duration: Duration::from_secs(DEFAULT_GRACE_SECS),
source: GraceSource::Default,
},
Some(s) => {
let trimmed = s.trim();
// Try parsing as i64 first so we can detect negatives.
if let Ok(val) = trimmed.parse::<i64>() {
if val <= 0 {
GracePeriod {
duration: Duration::from_secs(MIN_GRACE_SECS),
source: GraceSource::ClampedLow {
raw: val,
clamped_to: MIN_GRACE_SECS,
},
}
} else if (val as u64) > MAX_GRACE_SECS {
GracePeriod {
duration: Duration::from_secs(MAX_GRACE_SECS),
source: GraceSource::ClampedHigh {
raw: val as u64,
clamped_to: MAX_GRACE_SECS,
},
}
} else {
GracePeriod {
duration: Duration::from_secs(val as u64),
source: GraceSource::Env(val as u64),
}
}
} else {
GracePeriod {
duration: Duration::from_secs(DEFAULT_GRACE_SECS),
source: GraceSource::Invalid {
raw: trimmed.to_string(),
fallback: DEFAULT_GRACE_SECS,
},
}
}
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_unset_uses_default() {
let gp = parse_grace_period_from(None);
assert_eq!(gp.duration, Duration::from_secs(DEFAULT_GRACE_SECS));
assert_eq!(gp.source, GraceSource::Default);
}

#[test]
fn test_valid_value_in_range() {
let gp = parse_grace_period_from(Some("60".into()));
assert_eq!(gp.duration, Duration::from_secs(60));
assert_eq!(gp.source, GraceSource::Env(60));
}

#[test]
fn test_valid_value_at_minimum() {
let gp = parse_grace_period_from(Some("1".into()));
assert_eq!(gp.duration, Duration::from_secs(1));
assert_eq!(gp.source, GraceSource::Env(1));
}

#[test]
fn test_valid_value_at_maximum() {
let gp = parse_grace_period_from(Some("300".into()));
assert_eq!(gp.duration, Duration::from_secs(300));
assert_eq!(gp.source, GraceSource::Env(300));
}

#[test]
fn test_zero_clamped_to_minimum() {
let gp = parse_grace_period_from(Some("0".into()));
assert_eq!(gp.duration, Duration::from_secs(MIN_GRACE_SECS));
assert!(matches!(
gp.source,
GraceSource::ClampedLow {
raw: 0,
clamped_to: MIN_GRACE_SECS
}
));
}

#[test]
fn test_negative_clamped_to_minimum() {
let gp = parse_grace_period_from(Some("-5".into()));
assert_eq!(gp.duration, Duration::from_secs(MIN_GRACE_SECS));
assert!(matches!(
gp.source,
GraceSource::ClampedLow {
raw: -5,
clamped_to: MIN_GRACE_SECS
}
));
}

#[test]
fn test_too_large_clamped_to_maximum() {
let gp = parse_grace_period_from(Some("9999".into()));
assert_eq!(gp.duration, Duration::from_secs(MAX_GRACE_SECS));
assert!(matches!(
gp.source,
GraceSource::ClampedHigh {
raw: 9999,
clamped_to: MAX_GRACE_SECS
}
));
}

#[test]
fn test_non_numeric_falls_back_to_default() {
let gp = parse_grace_period_from(Some("abc".into()));
assert_eq!(gp.duration, Duration::from_secs(DEFAULT_GRACE_SECS));
assert!(matches!(
gp.source,
GraceSource::Invalid {
raw,
fallback: DEFAULT_GRACE_SECS
} if raw == "abc"
));
}

#[test]
fn test_whitespace_trimmed() {
let gp = parse_grace_period_from(Some(" 45 ".into()));
assert_eq!(gp.duration, Duration::from_secs(45));
assert_eq!(gp.source, GraceSource::Env(45));
}

#[test]
fn test_float_string_falls_back_to_default() {
let gp = parse_grace_period_from(Some("30.5".into()));
assert_eq!(gp.duration, Duration::from_secs(DEFAULT_GRACE_SECS));
assert!(matches!(
gp.source,
GraceSource::Invalid {
raw,
fallback: DEFAULT_GRACE_SECS
} if raw == "30.5"
));
}
}
24 changes: 24 additions & 0 deletions diagnostic/build-db991709.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"generated_at": "2026-06-24T08:37:42.951175+00:00",
"commit": "db991709",
"diagnostic_logd": "diagnostic\\build-db991709.logd",
"diagnostic_logd_error": null,
"message_blocker": null,
"chunked": false,
"chunk_size_bytes": null,
"password": "b8d0f1af87947fb271dc",
"decrypt_command": "encryptly unpack diagnostic\\build-db991709.logd <outdir> --password b8d0f1af87947fb271dc",
"total_modules": 1,
"passed": 0,
"failed": 1,
"modules": [
{
"name": "backend",
"status": "FAIL",
"elapsed_seconds": 300.006,
"artifact": null,
"output": "BUILD TIMEOUT (300s)"
}
],
"pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic\\build-db991709.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
}
Binary file added diagnostic/build-db991709.logd
Binary file not shown.
52 changes: 52 additions & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,55 @@ Audit logs are retained for 365 days and include:
2. Update Kubernetes secret: `kubectl create secret tls tot-tls --cert=new.crt --key=new.key -n tent-production --dry-run=client -o yaml | kubectl apply -f -`
3. Restart services: `kubectl rollout restart deployment -n tent-production`
4. Verify new certificate: `openssl s_client -connect api.example.com:443 -servername api.example.com`

## Graceful Shutdown Configuration

### Shutdown Grace Period

The backend supports a configurable shutdown grace period that controls how
long the process waits for in-flight requests to complete before forcing
exit. This is set via the `TOT_SHUTDOWN_GRACE_SECS` environment variable.

| Setting | Value |
|---------|-------|
| Environment variable | `TOT_SHUTDOWN_GRACE_SECS` |
| Default | 30 seconds |
| Minimum | 1 second |
| Maximum | 300 seconds (5 minutes) |

### Behavior

| Input | Result |
|-------|--------|
| Unset | Uses the default (30s) |
| Valid integer in range (1–300) | Uses the parsed value |
| Zero or negative | Clamped to 1 second |
| Above 300 | Clamped to 300 seconds |
| Non-numeric (e.g. `abc`, `30.5`) | Falls back to 30 seconds |

The parser module is at `backend/src/shutdown_grace.rs` and returns a
`GracePeriod` struct with both the resolved `Duration` and a `GraceSource`
enum describing how the value was determined (default, env, clamped, or
invalid).

### Running the Tests

```bash
cd backend && cargo test shutdown_grace
```

### Kubernetes Integration

When running on Kubernetes, set the pod `terminationGracePeriodSeconds` to
be at least 10 seconds longer than `TOT_SHUTDOWN_GRACE_SECS` so the process
has time to shut down before the kubelet sends SIGKILL:

```yaml
spec:
terminationGracePeriodSeconds: 40 # grace period + 10s buffer
containers:
- name: backend
env:
- name: TOT_SHUTDOWN_GRACE_SECS
value: "30"
```