diff --git a/docs/architecture.md b/docs/architecture.md index 3b6ee59a..40ac5042 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,102 +1,232 @@ -# Architecture Diagram - -```mermaid -graph TB - Client[Client App] -->|HTTP/WS| Gateway[NestJS Gateway] - - Gateway --> Auth[Auth Module] - Gateway --> Users[Users Module] - Gateway --> Properties[Properties Module] - Gateway --> Transactions[Transactions Module] - Gateway --> Documents[Documents Module] - Gateway --> Search[Search Module] - Gateway --> Admin[Admin Module] - Gateway --> Fraud[Fraud Module] - Gateway --> Notifications[Notifications Module] - - Auth --> JWT[JWT Strategy] - Auth --> Google[Google OAuth] - Auth --> MFA[2FA / TOTP] - Auth --> RateLimit[Rate Limit Guard] - Auth --> APIKeys[API Key Management] - - Users --> Prisma[(Prisma Client)] - Properties --> Prisma - Transactions --> Prisma - Documents --> Prisma - Search --> Prisma - Admin --> Prisma - Fraud --> Prisma - - Prisma --> PostgreSQL[(PostgreSQL)] - - Properties --> Blockchain[Blockchain Service] - Blockchain --> Ethereum[Ethereum / Sepolia] - - Documents --> Uploads[File Uploads] - Uploads --> LocalFS[Local FS / S3] - - Notifications --> WebSocket[WebSocket Gateway] - Notifications --> Email[Email Service] - Notifications --> SMS[SMS Service] - - Fraud --> Email - Admin --> Backup[Backup Service] - Backup --> PgDump[pg_dump] - - Search --> Cache[(Redis Cache)] - RateLimit --> Cache - Cache --> CacheWarming[Cache Warming] - - Admin --> BullMQ[BullMQ Queues] - Transactions --> BullMQ - - Gateway --> GraphQL[GraphQL / Apollo] +# PropChain Backend Architecture + +## Tech Stack + +| Layer | Technology | +| ---------------- | ---------------------------------------- | +| Runtime | Node.js >= 18 | +| Framework | NestJS 10 | +| Language | TypeScript 5.3 | +| Database | PostgreSQL (via Prisma ORM 6.x) | +| Cache | Redis (via ioredis + cache-manager) | +| API | REST (Express) + GraphQL (Apollo Server) | +| Authentication | Passport.js (JWT + Google OAuth2) | +| Blockchain | Web3.js / Ethers.js 6 | +| Job Queue | BullMQ (Redis-backed) | +| Email | Nodemailer via @nestjs-modules/mailer | +| Image Processing | Sharp | +| PDF Generation | PDFKit | +| Documentation | Swagger / OpenAPI | +| Testing | Jest + Supertest | +| Monitoring | Prometheus (prom-client) | +| Realtime | Socket.IO (WebSockets) | + +## Directory Structure + +``` +propchain-backend/ +├── prisma/ +│ ├── schema.prisma # Database schema +│ ├── seed.ts # Seed script +│ └── migrations/ # Migration history +├── src/ +│ ├── main.ts # Application bootstrap +│ ├── app.module.ts # Root module +│ ├── app.controller.ts # Health check / root routes +│ ├── admin/ # Admin panel & management +│ ├── analytics/ # Search & property analytics +│ ├── audit/ # Audit logging +│ ├── auth/ # Auth (JWT, OAuth, guards, RBAC) +│ ├── backup/ # Database backup/restore +│ ├── blockchain/ # On-chain recording & verification +│ ├── cache/ # Redis caching layer +│ ├── commissions/ # Agent commission tracking +│ ├── common/ # Shared types, middleware, decorators +│ ├── config/ # Swagger, env validation +│ ├── content/ # Content management +│ ├── dashboard/ # Dashboard aggregations +│ ├── database/ # Prisma service & module +│ ├── documents/ # Document upload, versioning, signing +│ ├── duplicate-detection/ # Property duplicate detection +│ ├── email/ # Email sending +│ ├── email-digest/ # Digest emails +│ ├── favorites/ # User favorites/bookmarks +│ ├── fraud/ # Fraud detection & investigation +│ ├── integrations/ # Third-party adapters +│ ├── metrics/ # Prometheus metrics +│ ├── mortgage-calculator/ # Mortgage estimation +│ ├── neighborhoods/ # Neighborhoods, schools, amenities +│ ├── notifications/ # Push, in-app, SMS notifications +│ ├── open-house/ # Open house scheduling & RSVP +│ ├── properties/ # Core property CRUD & images +│ ├── property-comparison/ # Side-by-side comparison +│ ├── property-views/ # View tracking & analytics +│ ├── search/ # Full-text search & suggestions +│ ├── sessions/ # User session management +│ ├── support-tickets/ # Support ticket system +│ ├── tracking/ # Link click tracking +│ ├── transactions/ # Transaction lifecycle +│ ├── trust-score/ # User trust scoring +│ ├── types/ # Shared TypeScript types +│ ├── users/ # User profiles & preferences +│ ├── utils/ # Utility functions +│ ├── versioning/ # API versioning +│ └── webhooks/ # Webhook management +├── test/ # E2E tests +├── docs/ # Developer documentation +└── package.json +``` + +## Module Dependency Diagram + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ AppModule │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │ +│ │ Config │ │ Prisma │ │ GraphQL │ │ ScheduleModule │ │ +│ │ Module │ │ Module │ │ Module │ │ │ │ +│ └────┬─────┘ └────┬─────┘ └──────────┘ └──────────────────┘ │ +│ │ │ │ +│ ┌────▼──────────────▼────────────────────────────────────────────┐ │ +│ │ Core Infrastructure │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────────┐ │ │ +│ │ │ Cache │ │ Auth │ │Database │ │ Rate Limit │ │ │ +│ │ │ Module │ │ Module │ │ Module │ │ Service │ │ │ +│ │ └──────────┘ └────┬─────┘ └──────────┘ └──────────────┘ │ │ +│ └─────────────────────┼─────────────────────────────────────────┘ │ +│ │ │ +│ ┌─────────────────────▼─────────────────────────────────────────┐ │ +│ │ Feature Modules │ │ +│ │ │ │ +│ │ ┌───────────┐ ┌────────────┐ ┌──────────────────────┐ │ │ +│ │ │Properties │ │Transactions│ │ Documents │ │ │ +│ │ │ Module │──│ Module │──│ Module │ │ │ +│ │ └─────┬─────┘ └─────┬──────┘ └──────────────────────┘ │ │ +│ │ │ │ │ │ +│ │ ┌─────▼─────┐ ┌─────▼──────┐ ┌──────────────────────┐ │ │ +│ │ │Property │ │Commissions │ │ Neighborhoods │ │ │ +│ │ │ Images │ │ Module │ │ Module │ │ │ +│ │ │ Module │ └────────────┘ └──────────────────────┘ │ │ +│ │ └───────────┘ │ │ +│ │ │ │ +│ │ ┌───────────┐ ┌────────────┐ ┌──────────────────────┐ │ │ +│ │ │Favorites │ │ Search │ │ Notifications │ │ │ +│ │ │ Module │ │ Module │ │ Module │ │ │ +│ │ └───────────┘ └────────────┘ └──────────────────────┘ │ │ +│ │ │ │ +│ │ ┌───────────┐ ┌────────────┐ ┌──────────────────────┐ │ │ +│ │ │ Fraud │ │ Blockchain│ │ Analytics │ │ │ +│ │ │ Module │ │ Module │ │ Module │ │ │ +│ │ └───────────┘ └────────────┘ └──────────────────────┘ │ │ +│ │ │ │ +│ │ ┌───────────┐ ┌────────────┐ ┌──────────────────────┐ │ │ +│ │ │ Admin │ │ Backup │ │ Metrics │ │ │ +│ │ │ Module │ │ Module │ │ Module │ │ │ +│ │ └───────────┘ └────────────┘ └──────────────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Data Flow Overview + +### Request Lifecycle + +``` +Client Request + │ + ▼ +┌──────────────────┐ +│ Rate Limiting │ IP-based + user-based throttling +│ (Guard) │ +└────────┬─────────┘ + │ + ▼ +┌──────────────────┐ +│ JWT Auth │ Token validation, session check +│ (Guard) │ +└────────┬─────────┘ + │ + ▼ +┌──────────────────┐ +│ RBAC Guard │ Role + permission checking +└────────┬─────────┘ + │ + ▼ +┌──────────────────┐ +│ ValidationPipe │ DTO validation, whitelisting +│ (NestJS) │ +└────────┬─────────┘ + │ + ▼ +┌──────────────────┐ +│ Controller │ Route handling +└────────┬─────────┘ + │ + ▼ +┌──────────────────┐ +│ Service │ Business logic, authorization +└────────┬─────────┘ + │ + ┌────┴────┐ + ▼ ▼ +┌────────┐ ┌────────┐ +│ Prisma │ │ Redis │ +│ (PostgreSQL) │ Cache │ +└────────┘ └────────┘ ``` -## Request Flow - -```mermaid -sequenceDiagram - participant C as Client - participant G as NestJS Gateway - participant RL as Rate Limit Guard - participant AU as Auth Service - participant S as Feature Service - participant DB as PostgreSQL - participant BC as Blockchain - - C->>G: HTTP Request - G->>RL: canActivate() - RL->>RL: Check Redis counters - alt Rate limit exceeded - RL-->>C: 429 Too Many Requests - else Within limits - RL->>G: Allow - G->>AU: Validate JWT - AU->>AU: Verify token + roles - AU->>G: Authenticated context - G->>S: Handle request - S->>DB: Query/Mutate - DB-->>S: Result - alt Blockchain required - S->>BC: Record transaction - BC-->>S: Tx hash - end - S-->>C: Response - end +### Property Image Pipeline + ``` +Upload (multer/memory buffer) + │ + ▼ +┌──────────────────────┐ +│ Validate │ Size, mime type, per-property cap +└──────────┬───────────┘ + │ + ▼ +┌──────────────────────┐ +│ Sharp Pipeline │ Auto-rotate, resize, convert +│ ├── Full (1920px) │ WebP quality 85 +│ ├── Medium (800px) │ WebP quality 80 +│ └── Thumbnail (300px)│ WebP quality 75 +│ Strip EXIF metadata │ Privacy protection +└──────────┬───────────┘ + │ + ▼ +┌──────────────────────┐ +│ Duplicate Detection │ Perceptual hash (SHA-256 prefix) +└──────────┬───────────┘ + │ + ▼ +┌──────────────────────┐ +│ Persist to DB + Disk │ PropertyImage record + files on disk +└──────────────────────┘ +``` + +### Transaction Lifecycle + +``` +PENDING → UNDER_CONTRACT → COMPLETED + │ │ + └── CANCELLED ─┘ + +Each state change: + 1. Validate transition (status machine) + 2. Update property status + 3. Record transaction history + 4. Emit notifications + 5. Optional: record on blockchain +``` + +## Key Design Decisions -## Module Relationships - -| Module | Depends On | Provides | -|--------|-----------|----------| -| Auth | Users, JWT, Redis | Authentication, 2FA, API keys | -| Users | Prisma | User CRUD, profiles | -| Properties | Prisma, Blockchain, Geocoding | Property listings, images | -| Transactions | Prisma, Blockchain | Transaction lifecycle | -| Documents | Prisma, Uploads | Document management, signing | -| Search | Prisma, Redis | Full-text search, facets, autocomplete | -| Fraud | Prisma, Email | Fraud detection, alerts, auto-block | -| Admin | Prisma, Backup, BullMQ | Admin ops, backups, reports | -| Notifications | WebSocket, Email, SMS | Real-time + async notifications | +1. **Dual API Surface**: REST (primary) + GraphQL (queries/subscriptions) +2. **Prisma as Single Source of Truth**: All DB access through PrismaService +3. **Decorator-based RBAC**: `@RequirePermissions()` for endpoint authorization +4. **Event-Driven Notifications**: WebSocket gateway for real-time delivery +5. **Blockchain as Audit Trail**: Optional on-chain recording for transactions +6. **Image Variants via Sharp**: Three sizes generated at upload time, stored on disk +7. **Fraud Detection Pipeline**: Pattern matching with configurable severity levels diff --git a/package.json b/package.json index 9136e568..ac129317 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,6 @@ "jsonwebtoken": "^9.0.2", "keyv": "^5.6.0", "multer": "^2.2.0", - "nodemailer": "^8.0.7", "nodemailer": "^9.0.1", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", diff --git a/src/cache/cache-headers.interceptor.ts b/src/cache/cache-headers.interceptor.ts index 50aec430..a0b8f01a 100644 --- a/src/cache/cache-headers.interceptor.ts +++ b/src/cache/cache-headers.interceptor.ts @@ -4,16 +4,38 @@ import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nes import { Observable } from 'rxjs'; import { tap } from 'rxjs/operators'; +const IMAGE_CACHE_DURATIONS: Record = { + 'image/avif': 86400 * 30, + 'image/webp': 86400 * 7, + 'image/jpeg': 3600, + 'image/png': 3600, + 'image/gif': 3600, +}; + @Injectable() export class CacheHeadersInterceptor implements NestInterceptor { intercept(context: ExecutionContext, next: CallHandler): Observable { const res = context.switchToHttp().getResponse(); + const req = context.switchToHttp().getRequest(); const start = Date.now(); return next.handle().pipe( tap(() => { res.setHeader('X-Cache-Time', `${Date.now() - start}ms`); - res.setHeader('Cache-Control', 'public, max-age=60'); + + const contentType = res.getHeader('content-type') as string | undefined; + const isImageResponse = + contentType?.startsWith('image/') || + req.path?.includes('/uploads/'); + + if (isImageResponse) { + const format = contentType?.split(';')[0]?.trim() || 'image/jpeg'; + const maxAge = IMAGE_CACHE_DURATIONS[format] || 3600; + res.setHeader('Cache-Control', `public, max-age=${maxAge}`); + res.setHeader('Vary', 'Accept'); + } else { + res.setHeader('Cache-Control', 'public, max-age=60'); + } }), ); } diff --git a/src/documents/document-upload.service.spec.ts b/src/documents/document-upload.service.spec.ts index 9b8bd619..78e254c8 100644 --- a/src/documents/document-upload.service.spec.ts +++ b/src/documents/document-upload.service.spec.ts @@ -40,10 +40,10 @@ describe('DocumentUploadService', () => { const req: UploadRequest = { fileName: 'large.pdf', mimeType: 'application/pdf', - fileSizeBytes: 11 * 1024 * 1024, + fileSizeBytes: 30 * 1024 * 1024, }; expect(() => service.validate(req)).toThrow(BadRequestException); - expect(() => service.validate(req)).toThrow('File exceeds maximum allowed size'); + expect(() => service.validate(req)).toThrow('File exceeds maximum allowed size of 25 MB for application/pdf'); }); it('should throw BadRequestException for empty file name', () => { @@ -55,6 +55,34 @@ describe('DocumentUploadService', () => { expect(() => service.validate(req)).toThrow(BadRequestException); expect(() => service.validate(req)).toThrow('File name cannot be empty'); }); + + it('should throw BadRequestException for zero size file', () => { + const req: UploadRequest = { + fileName: 'empty.pdf', + mimeType: 'application/pdf', + fileSizeBytes: 0, + }; + expect(() => service.validate(req)).toThrow(BadRequestException); + expect(() => service.validate(req)).toThrow('File size must be greater than zero'); + }); + + it('should enforce type-specific size limits for images', () => { + const req: UploadRequest = { + fileName: 'big.png', + mimeType: 'image/png', + fileSizeBytes: 15 * 1024 * 1024, + }; + expect(() => service.validate(req)).toThrow(BadRequestException); + }); + + it('should enforce type-specific size limits for docs', () => { + const req: UploadRequest = { + fileName: 'big.doc', + mimeType: 'application/msword', + fileSizeBytes: 20 * 1024 * 1024, + }; + expect(() => service.validate(req)).toThrow(BadRequestException); + }); }); describe('prepareMetadata', () => { @@ -72,4 +100,120 @@ describe('DocumentUploadService', () => { expect(result).toHaveProperty('uploadedAt'); }); }); + + describe('validateMagicBytes', () => { + it('should validate PDF magic bytes', () => { + const pdfBuffer = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34]); + expect(service.validateMagicBytes(pdfBuffer, 'application/pdf')).toBe(true); + }); + + it('should reject wrong magic bytes', () => { + const buf = Buffer.from([0x00, 0x00, 0x00, 0x00]); + expect(service.validateMagicBytes(buf, 'application/pdf')).toBe(false); + }); + + it('should validate JPEG magic bytes', () => { + const jpegBuffer = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]); + expect(service.validateMagicBytes(jpegBuffer, 'image/jpeg')).toBe(true); + }); + + it('should validate PNG magic bytes', () => { + const pngBuffer = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + expect(service.validateMagicBytes(pngBuffer, 'image/png')).toBe(true); + }); + + it('should validate WebP magic bytes', () => { + const webpBuffer = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00]); + expect(service.validateMagicBytes(webpBuffer, 'image/webp')).toBe(true); + }); + + it('should return true for unknown MIME types', () => { + const buf = Buffer.from([0x00, 0x00]); + expect(service.validateMagicBytes(buf, 'application/octet-stream')).toBe(true); + }); + + it('should handle buffer shorter than signature', () => { + const shortBuf = Buffer.from([0x25]); + expect(service.validateMagicBytes(shortBuf, 'application/pdf')).toBe(false); + }); + }); + + describe('sanitizeFilename', () => { + it('should remove path traversal sequences', () => { + expect(service.sanitizeFilename('../../../etc/passwd')).not.toContain('..'); + }); + + it('should remove null bytes', () => { + expect(service.sanitizeFilename('file\x00name.pdf')).not.toContain('\x00'); + }); + + it('should remove slashes', () => { + expect(service.sanitizeFilename('path/to/file.pdf')).not.toContain('/'); + }); + + it('should replace special characters with underscores', () => { + const result = service.sanitizeFilename('hello world!@#.pdf'); + expect(result).not.toContain(' '); + expect(result).not.toContain('!'); + }); + + it('should handle empty input with fallback', () => { + const result = service.sanitizeFilename('...'); + expect(result).toMatch(/^upload_\d+$/); + }); + + it('should lowercase output', () => { + expect(service.sanitizeFilename('FILE.PDF')).toBe('file.pdf'); + }); + }); + + describe('validateFileSize', () => { + it('should pass for files within limits', () => { + const buf = Buffer.alloc(1024); + expect(() => service.validateFileSize(buf, 'application/pdf')).not.toThrow(); + }); + + it('should throw for oversized files', () => { + const buf = Buffer.alloc(30 * 1024 * 1024); + expect(() => service.validateFileSize(buf, 'application/pdf')).toThrow(BadRequestException); + }); + }); + + describe('scanForThreats', () => { + it('should detect script tags', () => { + const buf = Buffer.from(''); + const result = service.scanForThreats(buf); + expect(result.safe).toBe(false); + }); + + it('should detect javascript protocol', () => { + const buf = Buffer.from('javascript:void(0)'); + const result = service.scanForThreats(buf); + expect(result.safe).toBe(false); + }); + + it('should detect iframe tags', () => { + const buf = Buffer.from('