feat: implement circuit breaker, retries and graceful degradation

package.json

@@ -54,7 +54,7 @@
     "@nestjs/platform-express": "^10.3.0",
     "@nestjs/schedule": "^4.0.0",
     "@nestjs/swagger": "^7.2.0",
-    "@nestjs/terminus": "^10.2.3",
+    "@nestjs/terminus": "^10.3.0",
     "@nestjs/throttler": "^5.1.1",
     "@nestjs/typeorm": "^10.0.2",
     "@nomicfoundation/hardhat-toolbox": "^4.0.0",
@@ -85,6 +85,7 @@
     "moment": "^2.29.4",
     "multer": "^2.0.2",
     "nest-winston": "^1.10.2",
+    "opossum": "^9.0.0",
     "passport": "^0.7.0",
     "passport-custom": "^1.1.1",
     "passport-jwt": "^4.0.1",
@@ -140,7 +141,6 @@
     "tsconfig-paths": "^4.2.0",
     "typescript": "^5.3.2"
   },
-
   "engines": {
     "node": ">=18.0.0",
     "npm": ">=8.0.0"

src/common/errors/custom.exceptions.ts

@@ -125,3 +125,9 @@ export class OperationNotAllowedException extends BaseCustomException {
     super(ErrorCode.OPERATION_NOT_ALLOWED, message, undefined, HttpStatus.FORBIDDEN);
   }
 }
+
+export class ServiceUnavailableException extends BaseCustomException {
+  constructor(message?: string) {
+    super(ErrorCode.SERVICE_UNAVAILABLE, message, undefined, HttpStatus.SERVICE_UNAVAILABLE);
+  }
+}

src/common/errors/error.codes.ts

@@ -59,6 +59,10 @@ export enum ErrorCode {
   BUSINESS_RULE_VIOLATION = 'BUSINESS_RULE_VIOLATION',
   OPERATION_NOT_ALLOWED = 'OPERATION_NOT_ALLOWED',
   INVALID_STATE = 'INVALID_STATE',
+
+  SERVICE_UNAVAILABLE = 'SERVICE_UNAVAILABLE',
+  EXTERNAL_API_ERROR = 'EXTERNAL_API_ERROR',
+  CIRCUIT_OPEN = 'CIRCUIT_OPEN',
 }
 
 export const ErrorMessages: Record<ErrorCode, string> = {
@@ -110,4 +114,8 @@ export const ErrorMessages: Record<ErrorCode, string> = {
   [ErrorCode.BUSINESS_RULE_VIOLATION]: 'This operation violates business rules',
   [ErrorCode.OPERATION_NOT_ALLOWED]: 'This operation is not allowed',
   [ErrorCode.INVALID_STATE]: 'The resource is in an invalid state for this operation',
+
+  [ErrorCode.SERVICE_UNAVAILABLE]: 'The requested service is temporarily unavailable.',
+  [ErrorCode.CIRCUIT_OPEN]: 'Circuit breaker is open. Please try again later.',
+  [ErrorCode.EXTERNAL_API_ERROR]: 'An error occurred while communicating with an external service.',
 };

src/common/utils/resilence.util.ts

@@ -0,0 +1,39 @@
+import * as CircuitBreaker from 'opossum';
+import { InternalServerErrorException } from '@nestjs/common';
+
+export async function withResilience<T>(
+  action: () => Promise<T>,
+  options: {
+    name: string;
+    retries?: number;
+    fallback?: (err: any) => T | Promise<T>;
+  },
+): Promise<T> {
+  const breaker = new CircuitBreaker(action, {
+    timeout: 5000,
+    errorThresholdPercentage: 50,
+    resetTimeout: 30000,
+  });
+
+  breaker.fallback(
+    options.fallback ||
+      (err => {
+        throw new InternalServerErrorException(`${options.name} failed and no fallback available.`);
+      }),
+  );
+  let attempt = 0;
+  const maxRetries = options.retries || 3;
+
+  while (attempt < maxRetries) {
+    try {
+      return await breaker.fire();
+    } catch (error) {
+      attempt++;
+      if (attempt >= maxRetries || breaker.opened) {
+        throw error;
+      }
+      const delay = Math.pow(2, attempt) * 1000;
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+}

src/health/health.controller.ts

@@ -1,6 +1,6 @@
 import { Controller, Get } from '@nestjs/common';
 import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
-import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
+import { HealthCheck, HealthCheckService, HttpHealthIndicator } from '@nestjs/terminus';
 import { DatabaseHealthIndicator } from './indicators/database.health';
 import { RedisHealthIndicator } from './indicators/redis.health';
 import { BlockchainHealthIndicator } from './indicators/blockchain.health';
@@ -10,6 +10,7 @@ import { BlockchainHealthIndicator } from './indicators/blockchain.health';
 export class HealthController {
   constructor(
     private health: HealthCheckService,
+    private http: HttpHealthIndicator,
     private dbHealth: DatabaseHealthIndicator,
     private redisHealth: RedisHealthIndicator,
     private blockchainHealth: BlockchainHealthIndicator,
@@ -21,7 +22,11 @@ export class HealthController {
   @ApiResponse({ status: 200, description: 'Service is healthy' })
   @ApiResponse({ status: 503, description: 'Service is unhealthy' })
   check() {
-    return this.health.check([() => this.dbHealth.isHealthy('database'), () => this.redisHealth.isHealthy('redis')]);
+    return this.health.check([
+      () => this.dbHealth.isHealthy('database'), 
+      () => this.redisHealth.isHealthy('redis'),
+      () => this.http.pingCheck('valuation-provider', 'https://api.valuation-service.com/v1/health'),
+    ]);
   }
 
   @Get('detailed')

src/valuation/valuation.service.ts

@@ -4,6 +4,7 @@ import { PrismaService } from '../database/prisma/prisma.service';
 import axios from 'axios';
 import { Decimal } from '@prisma/client/runtime/library';
 import { CacheService } from '../common/services/cache.service';
+import { withResilience } from 'src/common/utils/resilence.util';
 
 export interface PropertyFeatures {
   id?: string;
@@ -286,19 +287,15 @@ export class ValuationService {
   private normalizeFeatures(features: PropertyFeatures): PropertyFeatures {
     // Normalize location: trim and lowercase
     const location = features.location ? features.location.trim().toLowerCase() : '';
-    
+
     // Convert string values to numbers where appropriate
-    const bedrooms = typeof features.bedrooms === 'string' ? 
-      parseInt(features.bedrooms, 10) : features.bedrooms;
-    const bathrooms = typeof features.bathrooms === 'string' ? 
-      parseFloat(features.bathrooms) : features.bathrooms;
-    const squareFootage = typeof features.squareFootage === 'string' ? 
-      parseInt(features.squareFootage, 10) : features.squareFootage;
-    const yearBuilt = typeof features.yearBuilt === 'string' ? 
-      parseInt(features.yearBuilt, 10) : features.yearBuilt;
-    const lotSize = typeof features.lotSize === 'string' ? 
-      parseFloat(features.lotSize) : features.lotSize;
-
+    const bedrooms = typeof features.bedrooms === 'string' ? parseInt(features.bedrooms, 10) : features.bedrooms;
+    const bathrooms = typeof features.bathrooms === 'string' ? parseFloat(features.bathrooms) : features.bathrooms;
+    const squareFootage =
+      typeof features.squareFootage === 'string' ? parseInt(features.squareFootage, 10) : features.squareFootage;
+    const yearBuilt = typeof features.yearBuilt === 'string' ? parseInt(features.yearBuilt, 10) : features.yearBuilt;
+    const lotSize = typeof features.lotSize === 'string' ? parseFloat(features.lotSize) : features.lotSize;
+
     return {
       ...features,
       location,
@@ -747,4 +744,30 @@ export class ValuationService {
       throw error;
     }
   }
+
+  async getPropertyValuation(propertyId: string) {
+    return withResilience(() => this.callExternalValuationApi(propertyId), {
+      name: 'ValuationAPI',
+      retries: 3,
+      fallback: async err => {
+        this.logger.warn(`Valuation API failed for ${propertyId}, using fallback. Error: ${err.message}`);
+        const lastPrice = await this.prisma.propertyValuation.findFirst({
+          where: { propertyId },
+          orderBy: { createdAt: 'desc' },
+        });
+        return lastPrice || { value: 0, status: 'ESTIMATED' };
+      },
+    });
+  }
+
+  private async callExternalValuationApi(propertyId: string) {
+    const apiKey = this.configService.get<string>('VALUATION_API_KEY');
+    const apiUrl = this.configService.get<string>('VALUATION_API_URL');
+
+    const response = await axios.get(`${apiUrl}/valuation/${propertyId}`, {
+      headers: { 'X-API-KEY': apiKey },
+    });
+
+    return response.data;
+  }
 }