Merge pull request #255 from dDevAhmed/feature/issue-253-scheduled-task-monitoring

RUKAYAT-CODER · web-flow · commit 54505aa4dd24 · 2026-03-30T11:30:11.000+01:00
feat(monitoring): add scheduled task tracking, timeout/missed alerts, and dashboard
diff --git a/src/backup/backup.service.ts b/src/backup/backup.service.ts
@@ -13,11 +13,18 @@ import { BackupResponseDto } from './dto/backup-response.dto';
 import { BackupJobData } from './interfaces/backup.interfaces';
 import { AlertingService } from '../monitoring/alerting/alerting.service';
 import { MetricsCollectionService } from '../monitoring/metrics/metrics-collection.service';
+import {
+  ScheduledTaskConfig,
+  ScheduledTaskMonitoringService,
+} from '../monitoring/scheduled-task-monitoring.service';
 
 @Injectable()
 export class BackupService {
   private readonly logger = new Logger(BackupService.name);
   private readonly retentionDays: number;
+  private readonly scheduledTaskRetryLimit: number;
+  private readonly scheduledTaskRetryDelayMs: number;
+  private readonly scheduledTaskTimeoutMs: number;
 
   constructor(
     @InjectRepository(BackupRecord)
@@ -27,8 +34,32 @@ export class BackupService {
     private readonly configService: ConfigService,
     private readonly alertingService: AlertingService,
     private readonly metricsService: MetricsCollectionService,
+    private readonly scheduledTaskMonitoringService: ScheduledTaskMonitoringService,
   ) {
     this.retentionDays = this.configService.get<number>('BACKUP_RETENTION_DAYS', 30);
+    this.scheduledTaskRetryLimit = this.configService.get<number>(
+      'BACKUP_SCHEDULED_TASK_RETRY_LIMIT',
+      2,
+    );
+    this.scheduledTaskRetryDelayMs = this.configService.get<number>(
+      'BACKUP_SCHEDULED_TASK_RETRY_DELAY_MS',
+      10000,
+    );
+    this.scheduledTaskTimeoutMs = this.configService.get<number>(
+      'BACKUP_SCHEDULED_TASK_TIMEOUT_MS',
+      30 * 60 * 1000,
+    );
+
+    this.scheduledTaskMonitoringService.registerTask('weekly-database-backup', {
+      expectedIntervalMs: 7 * 24 * 60 * 60 * 1000,
+      timeoutMs: this.scheduledTaskTimeoutMs,
+      maxRetries: this.scheduledTaskRetryLimit,
+    });
+    this.scheduledTaskMonitoringService.registerTask('cleanup-expired-backups', {
+      expectedIntervalMs: 24 * 60 * 60 * 1000,
+      timeoutMs: this.scheduledTaskTimeoutMs,
+      maxRetries: this.scheduledTaskRetryLimit,
+    });
   }
 
   /**
@@ -39,58 +70,59 @@ export class BackupService {
     timeZone: 'UTC',
   })
   async handleScheduledBackup(): Promise<void> {
-    this.logger.log('Starting scheduled weekly backup');
-
-    try {
-      const region =
-        (this.configService.get<string>('BACKUP_PRIMARY_REGION') as Region) || Region.US_EAST_1;
-      const databaseName = this.configService.get<string>('DB_DATABASE', 'teachlink');
-
-      const expiresAt = new Date();
-      expiresAt.setDate(expiresAt.getDate() + this.retentionDays);
-
-      const backupRecord = this.backupRepository.create({
-        backupType: BackupType.FULL,
-        status: BackupStatus.PENDING,
-        region,
-        databaseName,
-        storageKey: '',
-        expiresAt,
-        metadata: {
-          startTime: new Date(),
-        },
-      });
-
-      await this.backupRepository.save(backupRecord);
-
-      // Queue backup job
-      await this.backupQueue.add(
-        'create-backup',
-        {
-          backupRecordId: backupRecord.id,
+    await this.executeMonitoredScheduledTask(
+      'weekly-database-backup',
+      {
+        expectedIntervalMs: 7 * 24 * 60 * 60 * 1000,
+        timeoutMs: this.scheduledTaskTimeoutMs,
+        maxRetries: this.scheduledTaskRetryLimit,
+      },
+      async () => {
+        this.logger.log('Starting scheduled weekly backup');
+
+        const region =
+          (this.configService.get<string>('BACKUP_PRIMARY_REGION') as Region) || Region.US_EAST_1;
+        const databaseName = this.configService.get<string>('DB_DATABASE', 'teachlink');
+
+        const expiresAt = new Date();
+        expiresAt.setDate(expiresAt.getDate() + this.retentionDays);
+
+        const backupRecord = this.backupRepository.create({
           backupType: BackupType.FULL,
+          status: BackupStatus.PENDING,
           region,
           databaseName,
-        } as BackupJobData,
-        {
-          attempts: 3,
-          backoff: {
-            type: 'exponential',
-            delay: 10000,
+          storageKey: '',
+          expiresAt,
+          metadata: {
+            startTime: new Date(),
           },
-          timeout: 3600000, // 1 hour timeout
-        },
-      );
+        });
 
-      this.logger.log(`Scheduled backup ${backupRecord.id} queued`);
-    } catch (error) {
-      this.logger.error('Failed to initiate scheduled backup:', error);
-      this.alertingService.sendAlert(
-        'BACKUP_SCHEDULED_FAILED',
-        `Scheduled backup failed: ${error.message}`,
-        'CRITICAL',
-      );
-    }
+        await this.backupRepository.save(backupRecord);
+
+        // Queue backup job
+        await this.backupQueue.add(
+          'create-backup',
+          {
+            backupRecordId: backupRecord.id,
+            backupType: BackupType.FULL,
+            region,
+            databaseName,
+          } as BackupJobData,
+          {
+            attempts: 3,
+            backoff: {
+              type: 'exponential',
+              delay: 10000,
+            },
+            timeout: 3600000, // 1 hour timeout
+          },
+        );
+
+        this.logger.log(`Scheduled backup ${backupRecord.id} queued`);
+      },
+    );
   }
 
   /**
@@ -101,32 +133,96 @@ export class BackupService {
     timeZone: 'UTC',
   })
   async handleBackupCleanup(): Promise<void> {
-    this.logger.log('Starting backup cleanup job');
+    await this.executeMonitoredScheduledTask(
+      'cleanup-expired-backups',
+      {
+        expectedIntervalMs: 24 * 60 * 60 * 1000,
+        timeoutMs: this.scheduledTaskTimeoutMs,
+        maxRetries: this.scheduledTaskRetryLimit,
+      },
+      async () => {
+        this.logger.log('Starting backup cleanup job');
+
+        const expirationDate = new Date();
+        expirationDate.setDate(expirationDate.getDate() - this.retentionDays);
+
+        const expiredBackups = await this.backupRepository.find({
+          where: {
+            createdAt: LessThan(expirationDate),
+            status: BackupStatus.COMPLETED,
+          },
+        });
 
-    const expirationDate = new Date();
-    expirationDate.setDate(expirationDate.getDate() - this.retentionDays);
+        this.logger.log(`Found ${expiredBackups.length} expired backups to cleanup`);
 
-    const expiredBackups = await this.backupRepository.find({
-      where: {
-        createdAt: LessThan(expirationDate),
-        status: BackupStatus.COMPLETED,
+        for (const backup of expiredBackups) {
+          await this.backupQueue.add(
+            'delete-backup',
+            { backupRecordId: backup.id },
+            {
+              attempts: 3,
+              backoff: { type: 'exponential', delay: 5000 },
+            },
+          );
+        }
       },
+    );
+  }
+
+  private async executeMonitoredScheduledTask(
+    taskName: string,
+    config: ScheduledTaskConfig,
+    taskRunner: () => Promise<void>,
+  ): Promise<void> {
+    const executionId = this.scheduledTaskMonitoringService.startExecution(taskName, config, {
+      source: BackupService.name,
     });
 
-    this.logger.log(`Found ${expiredBackups.length} expired backups to cleanup`);
+    const maxAttempts = (config.maxRetries || 0) + 1;
 
-    for (const backup of expiredBackups) {
-      await this.backupQueue.add(
-        'delete-backup',
-        { backupRecordId: backup.id },
-        {
-          attempts: 3,
-          backoff: { type: 'exponential', delay: 5000 },
-        },
-      );
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+      try {
+        await taskRunner();
+        this.scheduledTaskMonitoringService.markSuccess(executionId, {
+          attempt,
+          maxAttempts,
+          retriesUsed: attempt - 1,
+        });
+        return;
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        const shouldRetry = attempt < maxAttempts;
+
+        this.logger.error(
+          `Scheduled task ${taskName} failed on attempt ${attempt}/${maxAttempts}: ${errorMessage}`,
+          error instanceof Error ? error.stack : undefined,
+        );
+
+        if (shouldRetry) {
+          this.scheduledTaskMonitoringService.recordRetry(taskName, attempt, maxAttempts - 1, errorMessage);
+          await this.delay(this.scheduledTaskRetryDelayMs);
+          continue;
+        }
+
+        this.scheduledTaskMonitoringService.markFailure(executionId, errorMessage, {
+          attempt,
+          maxAttempts,
+          retriesUsed: attempt - 1,
+        });
+
+        this.alertingService.sendAlert(
+          'BACKUP_SCHEDULED_FAILED',
+          `Scheduled task ${taskName} failed after ${maxAttempts} attempt(s): ${errorMessage}`,
+          'CRITICAL',
+        );
+      }
     }
   }
 
+  private async delay(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   async getLatestBackup(region?: Region): Promise<BackupRecord | null> {
     const where: any = {
       status: BackupStatus.COMPLETED,
diff --git a/src/monitoring/monitoring.controller.ts b/src/monitoring/monitoring.controller.ts
@@ -1,15 +1,24 @@
 import { Controller, Get, Res } from '@nestjs/common';
 import { MetricsCollectionService } from './metrics/metrics-collection.service';
 import { Response } from 'express';
+import { ScheduledTaskMonitoringService } from './scheduled-task-monitoring.service';
 
 @Controller('metrics')
 export class MonitoringController {
-  constructor(private readonly metricsService: MetricsCollectionService) {}
+  constructor(
+    private readonly metricsService: MetricsCollectionService,
+    private readonly scheduledTaskMonitoringService: ScheduledTaskMonitoringService,
+  ) {}
 
   @Get()
   async getMetrics(@Res() res: Response) {
     const metrics = await this.metricsService.getMetrics();
     res.set('Content-Type', this.metricsService.getRegistry().contentType);
     res.send(metrics);
   }
+
+  @Get('scheduled-tasks/dashboard')
+  getScheduledTasksDashboard() {
+    return this.scheduledTaskMonitoringService.getDashboard();
+  }
 }
diff --git a/src/monitoring/monitoring.module.ts b/src/monitoring/monitoring.module.ts
@@ -6,6 +6,7 @@ import { MetricsCollectionService } from './metrics/metrics-collection.service';
 import { PerformanceAnalysisService } from './performance/performance-analysis.service';
 import { OptimizationService } from './optimization/optimization.service';
 import { AlertingService } from './alerting/alerting.service';
+import { ScheduledTaskMonitoringService } from './scheduled-task-monitoring.service';
 
 @Module({
   imports: [ScheduleModule.forRoot()],
@@ -16,7 +17,8 @@ import { AlertingService } from './alerting/alerting.service';
     PerformanceAnalysisService,
     OptimizationService,
     AlertingService,
+    ScheduledTaskMonitoringService,
   ],
-  exports: [MetricsCollectionService, AlertingService],
+  exports: [MetricsCollectionService, AlertingService, ScheduledTaskMonitoringService],
 })
 export class MonitoringModule {}
diff --git a/src/monitoring/scheduled-task-monitoring.service.ts b/src/monitoring/scheduled-task-monitoring.service.ts
