From e4054148172b5f9413954e7df7914634a0c31dd4 Mon Sep 17 00:00:00 2001
From: frol-ai <frolvlad+ai@gmail.com>
Date: Tue, 21 Jul 2026 08:35:09 +0000
Subject: [PATCH] perf(nt-be): de-bloat apalis.jobs so the board queries are
 fast (not just quiet)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The board's slow queries (`/api/v1/overview`, `/api/v1/queues` ~1.5s) aren't slow
because of row count — testenv had only ~70k live rows — but because the table
is **bloated**: ~370 MB of dead tuples behind that live set, so the full-table
aggregates scan hundreds of MB. The queues churn ~1M rows/day and Postgres's
default autovacuum (`scale_factor` 0.2) is far too lax to keep up.

Fix the bloat at the source instead of hiding the warning:
- `tune_apalis_autovacuum` (in `setup_apalis`): make autovacuum aggressive on
  `apalis.jobs` — `vacuum_scale_factor` 0.05, `insert_scale_factor` 0.05 (this
  table is insert-heavy), lower thresholds, small cost delay — so dead tuples
  are reclaimed promptly (heap stays compact) and the visibility map stays fresh
  (the aggregates can use index-only scans over the narrow indexes added
  earlier, bypassing the heap).
- `VACUUM (ANALYZE) apalis.jobs` at the end of `apalis_prune` (hourly): reclaim
  the space the deletes leave and refresh the VM + stats. Plain VACUUM takes
  only a SHARE UPDATE EXCLUSIVE lock, so workers keep polling.

This supersedes the slow-statement threshold bump (#1162) for these queries —
it makes them genuinely fast rather than silencing the warning. One-time
`VACUUM FULL apalis.jobs` (ops, brief lock) reclaims the existing ~370 MB now;
going forward the settings above keep it from re-bloating.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FRWEKUFYCcmGGhwdDYd471
---
 nt-be/src/jobs/handlers.rs | 14 ++++++++++++++
 nt-be/src/jobs/mod.rs      | 29 +++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/nt-be/src/jobs/handlers.rs b/nt-be/src/jobs/handlers.rs
index 35b03226..14263cf3 100644
--- a/nt-be/src/jobs/handlers.rs
+++ b/nt-be/src/jobs/handlers.rs
@@ -575,6 +575,20 @@ pub async fn apalis_prune(_t: Tick, state: Data<Arc<AppState>>) -> Result<String
         }
     }
 
+    // Reclaim the space the deletes (and the churn) leave behind and refresh the
+    // visibility map + stats. Without this the table keeps hundreds of MB of
+    // dead tuples behind a small live set, and the board's full-table aggregate
+    // queries scan all of it. Plain `VACUUM` (not `FULL`) takes only a
+    // SHARE UPDATE EXCLUSIVE lock, so workers keep polling; it can't run inside
+    // a transaction, so it goes out on a pooled connection in autocommit. A
+    // failure here shouldn't fail the prune.
+    if let Err(e) = sqlx::query("VACUUM (ANALYZE) apalis.jobs")
+        .execute(&state.db_pool)
+        .await
+    {
+        tracing::warn!(error = %e, "VACUUM apalis.jobs after prune failed");
+    }
+
     Ok(format!(
         "pruned {total} terminal tasks older than {retention_hours}h"
     ))
diff --git a/nt-be/src/jobs/mod.rs b/nt-be/src/jobs/mod.rs
index aa4f18a4..e17c0686 100644
--- a/nt-be/src/jobs/mod.rs
+++ b/nt-be/src/jobs/mod.rs
@@ -233,9 +233,38 @@ async fn setup_apalis(pool: &PgPool) -> Result<(), sqlx::Error> {
     drop(conn);
 
     ensure_board_indexes(pool).await;
+    tune_apalis_autovacuum(pool).await;
     Ok(())
 }
 
+/// Makes autovacuum far more aggressive on `apalis.jobs`.
+///
+/// The queues churn ~1M rows/day (insert → Running → Done, then pruned), so at
+/// Postgres's default `autovacuum_vacuum_scale_factor` of 0.2 (vacuum only once
+/// 20% of the table is dead) the table bloats badly — on testenv it grew to
+/// ~370 MB of dead tuples behind only ~70k live rows, which made the board's
+/// full-table aggregate queries (`/api/v1/overview`, `/api/v1/queues`) scan
+/// hundreds of MB and take >1s. Aggressive settings keep dead tuples low (so the
+/// heap stays compact) and the visibility map fresh (so the board's aggregates
+/// can use index-only scans). `insert_scale_factor` matters because this table
+/// is insert-heavy — it triggers vacuums that keep the VM current even when few
+/// rows are updated. Idempotent; a failure here only degrades board latency, so
+/// log and carry on rather than block startup.
+async fn tune_apalis_autovacuum(pool: &PgPool) {
+    use sqlx::Executor as _;
+
+    let ddl = "ALTER TABLE apalis.jobs SET (\
+         autovacuum_vacuum_scale_factor = 0.05, \
+         autovacuum_vacuum_threshold = 500, \
+         autovacuum_analyze_scale_factor = 0.05, \
+         autovacuum_analyze_threshold = 500, \
+         autovacuum_vacuum_insert_scale_factor = 0.05, \
+         autovacuum_vacuum_cost_delay = 2)";
+    if let Err(e) = pool.execute(ddl).await {
+        tracing::warn!(error = %e, "failed to tune apalis.jobs autovacuum");
+    }
+}
+
 /// Adds composite indexes that the apalis-board queries need but the apalis
 /// schema doesn't ship.
 ///