Skip to content

Optimize frame pipeline latency #201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FreddyFunk merged 1 commit into from
Feb 3, 2026
Merged

Optimize frame pipeline latency #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions src/app/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -672,8 +672,11 @@ impl cosmic::Application for AppModel {
// Note: Preview continues during recording since VideoRecorder has its own pipeline
// Both can access the same PipeWire node simultaneously

// Give previous pipeline time to clean up (50ms should be enough)
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
// Give previous pipeline time to clean up
tokio::time::sleep(tokio::time::Duration::from_millis(
crate::constants::latency::PIPELINE_CLEANUP_DELAY_MS,
))
.await;

// Check cancel flag again after brief wait
if cancel_flag.load(std::sync::atomic::Ordering::Acquire) {
Expand All @@ -686,7 +689,9 @@ impl cosmic::Application for AppModel {
use crate::backends::camera::types::{CameraDevice, CameraFormat};

let (sender, mut receiver) =
cosmic::iced::futures::channel::mpsc::channel(100);
cosmic::iced::futures::channel::mpsc::channel(
crate::constants::latency::FRAME_CHANNEL_CAPACITY,
);

// Build device and format objects for backend
let device = CameraDevice {
Expand Down Expand Up @@ -748,43 +753,57 @@ impl cosmic::Application for AppModel {
}

// Wait for next frame with a timeout to periodically check cancellation
// Use 16ms timeout (~60fps) to reduce frame delivery jitter
// Timeout only affects cancel flag checking - frames arrive immediately when ready
match tokio::time::timeout(
tokio::time::Duration::from_millis(16),
tokio::time::Duration::from_millis(
crate::constants::latency::CANCEL_CHECK_INTERVAL_MS,
),
receiver.next(),
)
.await
{
Ok(Some(frame)) => {
// Drain any queued frames to get the most recent one (reduces latency)
let mut latest_frame = frame;
let mut drained_count = 0u32;
while let Ok(Some(newer_frame)) = receiver.try_next() {
latest_frame = newer_frame;
drained_count += 1;
}

frame_count += 1;
// Calculate frame latency (time from capture to subscription delivery)
let latency_us =
frame.captured_at.elapsed().as_micros();
latest_frame.captured_at.elapsed().as_micros();

if frame_count.is_multiple_of(30) {
debug!(
frame = frame_count,
width = frame.width,
height = frame.height,
width = latest_frame.width,
height = latest_frame.height,
latency_ms = latency_us as f64 / 1000.0,
drained = drained_count,
"Received frame from pipeline"
);
}

// Warn if latency exceeds 2 frame periods (>33ms at 60fps)
if latency_us > 33_000 {
// Warn if latency exceeds threshold
if latency_us
> crate::constants::latency::HIGH_LATENCY_WARNING_US
{
tracing::warn!(
frame = frame_count,
latency_ms = latency_us as f64 / 1000.0,
drained = drained_count,
"High frame latency detected - possible stuttering"
);
}

// Use try_send to avoid blocking the subscription when UI is busy
// Dropping frames is fine for live preview - we want the latest frame
match output
.try_send(Message::CameraFrame(Arc::new(frame)))
{
match output.try_send(Message::CameraFrame(Arc::new(
latest_frame,
))) {
Ok(_) => {
if frame_count.is_multiple_of(30) {
debug!(
Expand Down
6 changes: 4 additions & 2 deletions src/backends/camera/pipewire/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@ impl PipeWireBackend {
.as_ref()
.ok_or_else(|| BackendError::Other("No format set".to_string()))?;

// Create frame channel
let (sender, receiver) = cosmic::iced::futures::channel::mpsc::channel(100);
// Create frame channel (small capacity for low latency)
let (sender, receiver) = cosmic::iced::futures::channel::mpsc::channel(
crate::constants::latency::FRAME_CHANNEL_CAPACITY,
);

// Create pipeline
let pipeline = pipeline::PipeWirePipeline::new(device, format, sender.clone())?;
Expand Down
17 changes: 17 additions & 0 deletions src/constants.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,23 @@ pub mod timing {
pub const START_TIMEOUT_SECS: u64 = 5;
}

/// Frame latency optimization constants
pub mod latency {
/// Frame channel capacity (smaller = lower latency, more drops)
/// At 30fps, 4 frames = ~130ms max queue latency
pub const FRAME_CHANNEL_CAPACITY: usize = 4;

/// Cancel flag check interval in milliseconds
/// Higher values reduce overhead but slow camera switching response
pub const CANCEL_CHECK_INTERVAL_MS: u64 = 100;

/// High latency warning threshold in microseconds (33ms = 2 frames at 60fps)
pub const HIGH_LATENCY_WARNING_US: u128 = 33_000;

/// Pipeline cleanup delay in milliseconds before creating new pipeline
pub const PIPELINE_CLEANUP_DELAY_MS: u64 = 20;
}

/// Resolution labels for format picker
pub fn get_resolution_label(width: u32) -> Option<&'static str> {
match width {
Expand Down