#254 Graceful Shutdown Logic Is Duplicated Across All Three Services …

services/api-gateway/src/index.ts

@@ -28,7 +28,7 @@ import fastifyJwt from '@fastify/jwt';
 import rateLimit from '@fastify/rate-limit';
 import crypto from 'crypto';
 import { z } from 'zod';
-import { validateEnv, getPrismaLogLevels, setupPrismaQueryLogging, buildPrismaConnectionUrl, connectWithRetry, registerRequestId, createLoggerOptions, registerTracing } from '@bettapay/validation';
+import { validateEnv, getPrismaLogLevels, setupPrismaQueryLogging, buildPrismaConnectionUrl, connectWithRetry, registerRequestId, createLoggerOptions, registerTracing, registerGracefulShutdown } from '@bettapay/validation';
 import { createFxClient } from './clients/fx-client.js';
 import { createIndexerClient } from './clients/indexer-client.js';
 import {
@@ -944,27 +944,10 @@ fastify.get('/api/quote', async (request, reply) => {
   return proxyFxUpstream(request, reply, path);
 });
 
-// Graceful shutdown
-let shuttingDown = false;
-
-async function shutdown(signal: string) {
-  if (shuttingDown) return;
-  shuttingDown = true;
-
-  fastify.log.info(`Received ${signal}, shutting down gracefully...`);
-
-  try {
-    await fastify.close();
-    await prisma.$disconnect();
-    process.exit(0);
-  } catch (err) {
-    fastify.log.error(err, 'Error during shutdown');
-    process.exit(1);
-  }
-}
-
-process.on('SIGTERM', () => shutdown('SIGTERM'));
-process.on('SIGINT', () => shutdown('SIGINT'));
+// Graceful shutdown — delegated to the shared helper in @bettapay/validation.
+// It closes the HTTP server first, then disconnects Prisma, exiting 0 on
+// success or 1 on failure. A 30s force-exit timeout guards against a hang.
+registerGracefulShutdown({ fastify, prisma });
 
 const start = async () => {
   try {

services/fx-engine/src/index.ts

@@ -30,6 +30,7 @@ import {
   CurrencyCode,
   buildFxEngineHealthResponse,
   readServiceVersion,
+  registerGracefulShutdown,
 } from '@bettapay/validation';
 
 const env = validateEnv(process.env);
@@ -167,7 +168,6 @@ const fastify = Fastify({
 registerRequestId(fastify);
 redis = new Redis(env.REDIS_URL, { enableOfflineQueue: false });
 redis.on('error', (err) => fastify.log.warn({ err: err.message }, 'Redis error in fx-engine'));
-fastify.addHook('onClose', async () => { await redis.quit().catch(() => {}); });
 
 fastify.register(cors, {
   origin: env.ALLOWED_ORIGINS,
@@ -472,25 +472,10 @@ fastify.post<{ Body: VerifyQuoteRouteBody }>(
 
 // ── Start ──────────────────────────────────────────────────────────────────
 
-let shuttingDown = false;
-
-async function shutdown(signal: string) {
-  if (shuttingDown) return;
-  shuttingDown = true;
-
-  fastify.log.info(`Received ${signal}, shutting down gracefully...`);
-
-  try {
-    await fastify.close();
-    process.exit(0);
-  } catch (err) {
-    fastify.log.error(err, 'Error during shutdown');
-    process.exit(1);
-  }
-}
-
-process.on('SIGTERM', () => shutdown('SIGTERM'));
-process.on('SIGINT', () => shutdown('SIGINT'));
+// Graceful shutdown — delegated to the shared helper in @bettapay/validation.
+// The shared helper closes the HTTP server, then quits Redis, and now enforces
+// a 30s force-exit timeout (previously the FX engine had none).
+registerGracefulShutdown({ fastify, redis });
 
 const start = async () => {
   try {

services/indexer/src/index.ts

@@ -42,6 +42,7 @@ import {
   buildIndexerHealthResponse,
   readServiceVersion,
   createAuditLogger,
+  registerGracefulShutdown,
 } from '@bettapay/validation';
 import type { EventType } from '@bettapay/validation';
 
@@ -108,8 +109,6 @@ const webhookWorker = createWebhookWorker('indexer-webhooks', connectionParams,
   },
 });
 
-});
-
 const redisHealth = new Redis(env.REDIS_URL, { enableOfflineQueue: false });
 redisHealth.on('error', (err) => fastify.log.warn({ err: err.message }, '[Indexer] Redis health client error'));
 fastify.addHook('onClose', async () => {
@@ -543,12 +542,17 @@ const start = async () => {
   }
 };
 
-process.on('SIGTERM', async () => {
-  await prisma.$disconnect();
-  await webhookQueue.close();
-  await webhookWorker.close();
-  await fastify.close();
-  process.exit(0);
+// Graceful shutdown — delegated to the shared helper in @bettapay/validation.
+// It closes resources in the canonical order (server → worker → queue → prisma)
+// and now also wires up SIGINT, which the previous inline handler omitted.
+// The Redis health client is released via the server's onClose hook.
+registerGracefulShutdown({
+  fastify,
+  prisma,
+  bullmq: {
+    worker: webhookWorker,
+    queues: [webhookQueue],
+  },
 });
 
 if (process.env.NODE_ENV !== 'test') {

services/settlement-engine/src/index.ts

@@ -49,6 +49,7 @@ import {
   registerTracing,
   buildSettlementEngineHealthResponse,
   readServiceVersion,
+  registerGracefulShutdown,
 } from "@bettapay/validation";
 import type { PaginatedResponse, ApiResponse } from '@bettapay/shared-types';
 
@@ -600,73 +601,19 @@ fastify.post<{ Body: z.infer<typeof CreateSettlementBody> }>(
 // ============================================================================
 // GRACEFUL SHUTDOWN
 // ============================================================================
-
-let isShuttingDown = false;
-
-async function gracefulShutdown(signal: string): Promise<void> {
-  // Prevent multiple shutdown attempts
-  if (isShuttingDown) {
-    fastify.log.warn({ signal }, 'Shutdown already in progress, ignoring duplicate signal');
-    return;
-  }
-
-  isShuttingDown = true;
-  fastify.log.info({ signal }, 'Received shutdown signal, starting graceful shutdown');
-
-  // Set a timeout to force exit if shutdown hangs
-  const forceExitTimeout = setTimeout(() => {
-    fastify.log.error('Graceful shutdown timed out after 30 seconds, forcing exit');
-    process.exit(1);
-  }, 30000);
-
-  try {
-    // 1. Close Fastify server (stops accepting new connections)
-    fastify.log.info('Closing Fastify server...');
-    await fastify.close();
-    fastify.log.info('Fastify server closed');
-
-    // 2. Close BullMQ worker (drain and close gracefully)
-    fastify.log.info('Closing BullMQ worker...');
-    await worker.close();
-    fastify.log.info('BullMQ worker closed');
-
-    // 3. Close BullMQ queues
-    fastify.log.info('Closing BullMQ queues...');
-    await settlementQueue.close();
-    await settlementDLQ.close();
-    await webhookWorker.close();
-    await webhookQueue.close();
-    fastify.log.info('BullMQ queues closed');
-
-    // 4. Close Redis connection
-    fastify.log.info('Closing Redis connection...');
-    await redis.quit();
-    fastify.log.info('Redis connection closed');
-
-    // 5. Disconnect Prisma
-    fastify.log.info('Disconnecting Prisma...');
-    await prisma.$disconnect();
-    fastify.log.info('Prisma disconnected');
-
-    // Clear the force exit timeout
-    clearTimeout(forceExitTimeout);
-
-    fastify.log.info({ signal }, 'Graceful shutdown completed successfully');
-    process.exit(0);
-  } catch (error) {
-    fastify.log.error({ error, signal }, 'Error during graceful shutdown');
-    clearTimeout(forceExitTimeout);
-    process.exit(1);
-  }
-}
-
-// Register shutdown handlers for SIGTERM and SIGINT
-process.on('SIGTERM', () => {
-  void gracefulShutdown('SIGTERM');
-});
-
-process.on('SIGINT', () => {
-  void gracefulShutdown('SIGINT');
+//
+// Delegated to the shared helper in @bettapay/validation. It enforces the
+// canonical close order (server → workers → queues → redis → prisma) and a
+// 30s force-exit timeout, replacing the previous hand-rolled implementation.
+
+registerGracefulShutdown({
+  fastify,
+  prisma,
+  redis,
+  bullmq: {
+    worker: [worker, webhookWorker],
+    queues: [settlementQueue, settlementDLQ, webhookQueue],
+  },
 });
 
 // ============================================================================

shared/validation/index.ts

@@ -16,6 +16,7 @@ export * from './envAwareSchema.js';
 export * from './webhookSchema.js';
 export * from './health.js';
 export * from './audit.js';
+export * from './shutdown.js';
 import "dotenv/config";
 
 export function genReqId(req: FastifyRequest | IncomingMessage): string {

shared/validation/package.json

@@ -13,7 +13,7 @@
     "build": "tsc",
     "type-check": "tsc --noEmit",
     "pretest": "pnpm -F @bettapay/stellar-utils build",
-    "test": "cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm health.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm cors.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm prisma.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm genReqId.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm plugins.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm tracing.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm logger.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm schemas.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm envAwareSchema.test.ts"
+    "test": "cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm health.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm cors.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm prisma.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm genReqId.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm plugins.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm tracing.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm logger.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm schemas.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm envAwareSchema.test.ts && cross-env TS_NODE_TRANSPILE_ONLY=true node --loader ts-node/esm shutdown.test.ts"
   },
   "dependencies": {
     "@bettapay/stellar-utils": "workspace:^",