Merge pull request #174 from Mkalbani/feature/process-shutdown-hygiene

Feature/process shutdown hygiene

Author: greatest0fallt1me

README.md

@@ -118,4 +118,12 @@ The app validates all environment variables at startup using [Zod](https://zod.d
 | `LOG_LEVEL` | No | `info` | `trace` / `debug` / `info` / `warn` / `error` / `fatal` |
 | `GATEWAY_PROFILING_ENABLED` | No | `false` | Enable request profiling |
 
+## Production Shutdown Expectations
+
+- The server listens for `SIGTERM` and `SIGINT` and performs a graceful shutdown.
+- On shutdown, it stops accepting new HTTP requests, waits for active connections to finish, and closes database resources.
+- A 30 second timeout is enforced for in-flight connections; lingering sockets are destroyed to prevent hung termination.
+- Shutdown hooks are registered with `process.once(...)` to avoid duplicate execution during restarts.
+- The dev workflow (`npm run dev` with `tsx watch`) is preserved. Restarts trigger the same graceful path instead of abrupt termination.
+
 This repo is part of [Callora](https://github.com/your-org/callora). Frontend: `callora-frontend`. Contracts: `callora-contracts`.

package-lock.json

@@ -18,6 +18,8 @@ export const pool = new Pool({
   connectionTimeoutMillis: config.dbPool.connectionTimeoutMillis,
 });
 
+let poolClosed = false;
+
 /**
  * Convenience helper that proxies to pool.query for simple one-off queries.
  */
@@ -42,4 +44,12 @@ export async function checkDbHealth(): Promise<{ ok: boolean; error?: string }>
       error: error instanceof Error ? error.message : 'Unknown database error',
     };
   }
+}
+
+export async function closePgPool(): Promise<void> {
+  if (poolClosed) {
+    return;
+  }
+  await pool.end();
+  poolClosed = true;
 }

src/db.ts

@@ -5,6 +5,7 @@ import { readFileSync } from 'fs';
 import { join } from 'path';
 
 const logger = console;
+let sqliteClosed = false;
 
 // Create SQLite database instance
 const sqlite = new Database('./database.db');
@@ -61,7 +62,11 @@ export async function initializeDb() {
 
 // Graceful shutdown
 // Export close function for graceful shutdown
-export function closeDb() {
+export async function closeDb(): Promise<void> {
+  if (sqliteClosed) {
+    return;
+  }
   sqlite.close();
+  sqliteClosed = true;
 }
 export { schema };

src/db/index.ts

@@ -1,6 +1,7 @@
 /// <reference types="jest" />
 import request from 'supertest';
-import app from './index.js';
+import type { Server } from 'http';
+import app, { createGracefulShutdownHandler } from './index.js';
 
 jest.mock('./db/index.js', () => ({
   db: {},
@@ -14,3 +15,48 @@ describe('Health API', () => {
     expect(response.body.status).toBe('ok');
   });
 });
+
+describe('graceful shutdown', () => {
+  it('closes server and database resources', async () => {
+    const closeServer = jest.fn((callback: (err?: Error) => void) => callback());
+    const closeDatabase = jest.fn(async () => Promise.resolve());
+    const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() };
+
+    const shutdown = createGracefulShutdownHandler({
+      server: { close: closeServer } as unknown as Server,
+      activeConnections: new Set(),
+      closeDatabase,
+      logger,
+      timeoutMs: 50,
+    });
+
+    await expect(shutdown('SIGTERM')).resolves.toBe(0);
+    expect(closeServer).toHaveBeenCalledTimes(1);
+    expect(closeDatabase).toHaveBeenCalledTimes(1);
+  });
+
+  it('reuses in-flight shutdown promise on repeated signals', async () => {
+    let closeCallback: ((err?: Error) => void) | undefined;
+    const closeServer = jest.fn((callback: (err?: Error) => void) => {
+      closeCallback = callback;
+    });
+    const closeDatabase = jest.fn(async () => Promise.resolve());
+
+    const shutdown = createGracefulShutdownHandler({
+      server: { close: closeServer } as unknown as Server,
+      activeConnections: new Set(),
+      closeDatabase,
+      timeoutMs: 50,
+    });
+
+    const first = shutdown('SIGTERM');
+    const second = shutdown('SIGINT');
+
+    expect(closeServer).toHaveBeenCalledTimes(1);
+    closeCallback?.();
+
+    await expect(first).resolves.toBe(0);
+    await expect(second).resolves.toBe(0);
+    expect(closeDatabase).toHaveBeenCalledTimes(1);
+  });
+});

src/index.test.ts

@@ -1,11 +1,14 @@
 import './config/env.js'
 import express from 'express';
 import { initializeDb, closeDb } from './db/index.js';
-import { type AuthenticatedLocals } from './middleware/requireAuth.js';
+import { closePgPool } from './db.js';
+import { closeDbPool } from './config/health.js';
+import { disconnectPrisma } from './lib/prisma.js';
 import { errorHandler } from './middleware/errorHandler.js';
 import { createGatewayIpAllowlist } from './middleware/ipAllowlist.js';
 import type { Response } from 'express';
 import type { Socket } from 'net';
+import type { Server } from 'http';
 
 import { createDeveloperRouter } from './routes/developerRoutes.js';
 import { createGatewayRouter } from './routes/gatewayRoutes.js';
@@ -86,6 +89,15 @@ if (isDirectExecution) {
 
   const PORT = config.port;
 
+  const closeAllDataResources = async () => {
+    await closeDb();
+    await Promise.allSettled([
+      closePgPool(),
+      disconnectPrisma(),
+      closeDbPool(),
+    ]);
+  };
+
   // Initialize database and start server
   async function startServer() {
     try {
@@ -103,52 +115,21 @@ if (isDirectExecution) {
         socket.once('close', () => activeConnections.delete(socket));
       });
 
-      async function gracefulShutdown(signal: string) {
-        console.log(`\n[shutdown] Received ${signal}. Starting graceful shutdown...`);
-
-        // 1. Stop accepting new requests
-        server.close(() => {
-          console.log('[shutdown] HTTP server closed. No new requests accepted.');
-        });
+      const gracefulShutdown = createGracefulShutdownHandler({
+        server,
+        activeConnections,
+        closeDatabase: closeAllDataResources,
+      });
 
-        // 2. Wait for in-flight requests to finish (max 30s)
-        const TIMEOUT_MS = 30_000;
-        const deadline = setTimeout(() => {
-          console.warn('[shutdown] Timeout reached. Forcing exit.');
-          process.exit(1);
-        }, TIMEOUT_MS);
-        deadline.unref();
-
-        // 3. Wait until all active connections are gone
-        await new Promise<void>((resolve) => {
-          if (activeConnections.size === 0) return resolve();
-          console.log(`[shutdown] Waiting for ${activeConnections.size} in-flight connection(s)...`);
-          const interval = setInterval(() => {
-            if (activeConnections.size === 0) {
-              clearInterval(interval);
-              resolve();
-            }
-          }, 200);
+      const onSignal = (signal: NodeJS.Signals) => {
+        void gracefulShutdown(signal).then((exitCode) => {
+          process.exit(exitCode);
         });
-
-        // 4. Close the database
-        console.log('[shutdown] Closing database...');
-        try {
-          closeDb();
-          console.log('[shutdown] Database closed.');
-        } catch (err) {
-          console.error('[shutdown] Error closing database:', err);
-        }
-
-        // 5. Exit cleanly
-        console.log('[shutdown] Shutdown complete. Exiting.');
-        clearTimeout(deadline);
-        process.exit(0);
-      }
+      };
 
       // Register shutdown signals
-      process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
-      process.on('SIGINT',  () => gracefulShutdown('SIGINT'));
+      process.once('SIGTERM', () => onSignal('SIGTERM'));
+      process.once('SIGINT', () => onSignal('SIGINT'));
 
     } catch (error) {
       console.error('Failed to start server:', error);

src/index.ts

@@ -15,6 +15,13 @@ function getPrismaClient(): PrismaClient {
   return prisma;
 }
 
+export async function disconnectPrisma(): Promise<void> {
+  if (!prisma) {
+    return;
+  }
+  await prisma.$disconnect();
+}
+
 export default new Proxy({} as PrismaClient, {
   get(_target, prop, receiver) {
     const client = getPrismaClient();