Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import { MortgageCalculatorModule } from './mortgage-calculator/mortgage-calcula
import { SupportTicketsModule } from './support-tickets/support-tickets.module';
import { AuditModule } from './audit/audit.module';
import { MetricsModule } from './metrics/metrics.module';
import { PropertyTaxModule } from './properties/tax/property-tax.module';

@Module({
imports: [
Expand Down Expand Up @@ -86,6 +87,7 @@ import { MetricsModule } from './metrics/metrics.module';
SupportTicketsModule,
AuditModule,
MetricsModule,
PropertyTaxModule,
],

controllers: [AppController],
Expand Down
24 changes: 23 additions & 1 deletion src/duplicate-detection/duplicate-detection.controller.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// @ts-nocheck

import { Body, Controller, Get, Param, Patch, Post, UseGuards } from '@nestjs/common';
import { Body, Controller, Get, Param, Patch, Post, Query, UseGuards } from '@nestjs/common';
import { DuplicateDetectionService } from './duplicate-detection.service';
import { CheckDuplicateDto, FlagForReviewDto, MergeDuplicateDto } from './dto/duplicate.dto';
import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
Expand Down Expand Up @@ -50,4 +50,26 @@ export class DuplicateDetectionController {
async resolveFlag(@Param('id') flagId: string) {
return this.duplicateDetectionService.resolveFlag(flagId);
}

@UseGuards(JwtAuthGuard)
@Post('batch')
async detectBatchDuplicates(@Body() body: { propertyIds: string[] }) {
return this.duplicateDetectionService.detectBatchDuplicates(body.propertyIds);
}

@UseGuards(JwtAuthGuard)
@Get('stats')
async getDuplicateStats() {
return this.duplicateDetectionService.getDuplicateStats();
}

@UseGuards(JwtAuthGuard)
@Get(':propertyId/nearby')
async findNearbyDuplicates(
@Param('propertyId') propertyId: string,
@Query('radius') radius?: string,
) {
const radiusMeters = radius ? parseInt(radius, 10) : 500;
return this.duplicateDetectionService.findNearbyDuplicates(propertyId, radiusMeters);
}
}
196 changes: 196 additions & 0 deletions src/duplicate-detection/duplicate-detection.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,202 @@ export class DuplicateDetectionService {
});
}

// ---------- Issue #936: Enhanced Duplicate Detection ----------

async detectTextSimilarity(
propertyA: { description?: string | null; features?: string[] | null },
propertyB: { description?: string | null; features?: string[] | null },
): Promise<{ score: number; matchedTerms: string[] }> {
const tokenize = (text: string | null | undefined): Set<string> => {
if (!text) return new Set();
return new Set(
text
.toLowerCase()
.replace(/[^a-z0-9\s]/g, '')
.split(/\s+/)
.filter((t) => t.length > 2),
);
};

const tokensA = tokenize(propertyA.description);
const tokensB = tokenize(propertyB.description);

const featuresA = new Set((propertyA.features || []).map((f) => f.toLowerCase()));
const featuresB = new Set((propertyB.features || []).map((f) => f.toLowerCase()));

const allTermsA = new Set([...tokensA, ...featuresA]);
const allTermsB = new Set([...tokensB, ...featuresB]);

const intersection = new Set([...allTermsA].filter((t) => allTermsB.has(t)));
const union = new Set([...allTermsA, ...allTermsB]);

const score = union.size === 0 ? 0 : Math.round((intersection.size / union.size) * 100);

return {
score,
matchedTerms: Array.from(intersection),
};
}

async findNearbyDuplicates(
propertyId: string,
radiusMeters: number = 500,
): Promise<any[]> {
const property = await this.prisma.property.findUnique({
where: { id: propertyId },
select: { latitude: true, longitude: true },
});

if (!property?.latitude || !property?.longitude) {
return [];
}

const latDelta = radiusMeters / 111000;
const lngDelta = radiusMeters / (111000 * Math.cos((property.latitude * Math.PI) / 180));

const nearby = await this.prisma.property.findMany({
where: {
id: { not: propertyId },
latitude: {
gte: property.latitude - latDelta,
lte: property.latitude + latDelta,
},
longitude: {
gte: property.longitude - lngDelta,
lte: property.longitude + lngDelta,
},
},
select: {
id: true,
title: true,
address: true,
city: true,
state: true,
zipCode: true,
price: true,
latitude: true,
longitude: true,
},
take: 20,
});

return nearby;
}

calculateConfidence(signals: {
addressMatch?: boolean;
imageSimilarity?: number;
textSimilarity?: number;
weights?: { address?: number; image?: number; text?: number };
}): number {
const w = {
address: signals.weights?.address ?? 0.4,
image: signals.weights?.image ?? 0.35,
text: signals.weights?.text ?? 0.25,
};

let score = 0;
if (signals.addressMatch) score += 100 * w.address;
if (signals.imageSimilarity != null) score += signals.imageSimilarity * w.image;
if (signals.textSimilarity != null) score += signals.textSimilarity * w.text;

return Math.round(Math.min(100, score));
}

async detectBatchDuplicates(
propertyIds: string[],
): Promise<Map<string, { matches: any[]; confidence: number }>> {
const results = new Map<string, { matches: any[]; confidence: number }>();

for (const propId of propertyIds) {
const property = await this.prisma.property.findUnique({
where: { id: propId },
include: {
owner: { select: { id: true, firstName: true, lastName: true } },
images: { select: { id: true, url: true }, take: 5 },
},
});

if (!property) continue;

const matches: any[] = [];

// Address match
const addressMatches = await this.prisma.property.findMany({
where: {
id: { not: propId },
address: { equals: property.address, mode: 'insensitive' },
city: { equals: property.city, mode: 'insensitive' },
state: { equals: property.state, mode: 'insensitive' },
zipCode: property.zipCode,
},
take: 5,
});

for (const m of addressMatches) {
matches.push({ id: m.id, type: 'ADDRESS', address: m.address });
}

// Nearby duplicates
const nearby = await this.findNearbyDuplicates(propId, 500);
for (const n of nearby) {
if (!matches.find((m: any) => m.id === n.id)) {
matches.push({ id: n.id, type: 'NEARBY', address: n.address });
}
}

const confidence = matches.length > 0
? this.calculateConfidence({ addressMatch: matches.some((m: any) => m.type === 'ADDRESS') })
: 0;

results.set(propId, { matches, confidence });
}

return results;
}

async getDuplicateStats(): Promise<{
total: number;
byStatus: Record<string, number>;
byType: Record<string, number>;
}> {
const total = await this.prisma.propertyDuplicate.count();

const records = await this.prisma.propertyDuplicate.findMany({
select: {
isMerged: true,
isResolved: true,
flaggedForReview: true,
duplicateType: true,
},
});

const byStatus: Record<string, number> = {
PENDING: 0,
REVIEWED: 0,
MERGED: 0,
DISMISSED: 0,
};

const byType: Record<string, number> = {};

for (const r of records) {
if (r.isMerged) {
byStatus.MERGED++;
} else if (r.isResolved) {
byStatus.DISMISSED++;
} else if (r.flaggedForReview) {
byStatus.PENDING++;
} else {
byStatus.REVIEWED++;
}

byType[r.duplicateType] = (byType[r.duplicateType] || 0) + 1;
}

return { total, byStatus, byType };
}

private async findSimilarImages(
hashes: string[],
excludeOwnerId: string,
Expand Down
127 changes: 127 additions & 0 deletions src/neighborhoods/neighborhoods.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,133 @@ export class NeighborhoodsService {
});
}

// ---------- Neighborhood Scoring (#935) ----------

private readonly defaultWeights = {
walkScore: 0.25,
transitScore: 0.2,
bikeScore: 0.1,
crimeIndex: 0.25,
schoolRating: 0.2,
};

async calculateCompositeScore(
neighborhoodId: string,
weights?: Record<string, number>,
): Promise<{ score: number; breakdown: Record<string, number | null> }> {
const neighborhood = await this.prisma.neighborhood.findUnique({
where: { id: neighborhoodId },
select: {
walkScore: true,
transitScore: true,
bikeScore: true,
crimeIndex: true,
schoolRating: true,
metadata: true,
},
});

if (!neighborhood) {
throw new NotFoundException(`Neighborhood ${neighborhoodId} not found`);
}

const w = { ...this.defaultWeights, ...weights };
const totalWeight = Object.values(w).reduce((s, v) => s + v, 0);
const normalized = totalWeight > 0 ? totalWeight : 1;

const normalizedWeights = {
walkScore: w.walkScore / normalized,
transitScore: w.transitScore / normalized,
bikeScore: w.bikeScore / normalized,
crimeIndex: w.crimeIndex / normalized,
schoolRating: w.schoolRating / normalized,
};

const breakdown: Record<string, number | null> = {
walkScore: neighborhood.walkScore,
transitScore: neighborhood.transitScore,
bikeScore: neighborhood.bikeScore,
crimeIndex: neighborhood.crimeIndex,
schoolRating: neighborhood.schoolRating,
};

let score = 0;
if (neighborhood.walkScore != null) {
score += neighborhood.walkScore * normalizedWeights.walkScore;
}
if (neighborhood.transitScore != null) {
score += neighborhood.transitScore * normalizedWeights.transitScore;
}
if (neighborhood.bikeScore != null) {
score += neighborhood.bikeScore * normalizedWeights.bikeScore;
}
if (neighborhood.crimeIndex != null) {
score += (100 - neighborhood.crimeIndex) * normalizedWeights.crimeIndex;
}
if (neighborhood.schoolRating != null) {
score += (neighborhood.schoolRating / 10) * 100 * normalizedWeights.schoolRating;
}

const finalScore = Math.round(Math.min(100, Math.max(1, score)));

const existingMetadata = (neighborhood.metadata as Record<string, any>) || {};
const scoreHistory = Array.isArray(existingMetadata.scoreHistory)
? existingMetadata.scoreHistory
: [];
scoreHistory.push({
score: finalScore,
weights: normalizedWeights,
breakdown,
calculatedAt: new Date().toISOString(),
});

await this.prisma.neighborhood.update({
where: { id: neighborhoodId },
data: {
metadata: {
...existingMetadata,
scoreHistory,
},
} as any,
});

return { score: finalScore, breakdown };
}

async getScoreHistory(neighborhoodId: string): Promise<any[]> {
await this.assertExists(neighborhoodId);
const neighborhood = await this.prisma.neighborhood.findUnique({
where: { id: neighborhoodId },
select: { metadata: true },
});
const metadata = (neighborhood!.metadata as Record<string, any>) || {};
return Array.isArray(metadata.scoreHistory) ? metadata.scoreHistory : [];
}

async rankNeighborhoods(
city: string,
weights?: Record<string, number>,
): Promise<Array<{ rank: number; neighborhoodId: string; name: string; score: number }>> {
const neighborhoods = await this.prisma.neighborhood.findMany({
where: { city },
select: { id: true, name: true },
});

const scored: Array<{ rank: number; neighborhoodId: string; name: string; score: number }> = [];

for (const n of neighborhoods) {
const { score } = await this.calculateCompositeScore(n.id, weights);
scored.push({ rank: 0, neighborhoodId: n.id, name: n.name, score });
}

scored.sort((a, b) => b.score - a.score);
scored.forEach((item, idx) => {
item.rank = idx + 1;
});

return scored;
}

private async assertExists(id: string) {
const found = await this.prisma.neighborhood.findUnique({
where: { id },
Expand Down
Loading
Loading