feat: prevent IP / alt IP conflicts

public/v1/components/examples.yaml

@@ -13,8 +13,8 @@ components:
               "city": "Ashburn",
               "asn": 14618,
               "network": "Amazon.com, Inc.",
-              "latitude": 39.0437,
-              "longitude": -77.4875
+              "latitude": 39.04,
+              "longitude": -77.49
             },
             "tags": [
               "aws-us-east-1",
@@ -127,8 +127,8 @@ components:
                 "state": null,
                 "city": "Auckland",
                 "asn": 61138,
-                "longitude": 174.76667,
-                "latitude": -36.86667,
+                "longitude": 174.77,
+                "latitude": -36.87,
                 "network": "Zappie Host LLC",
                 "tags": [
                   "datacenter-network"
@@ -169,8 +169,8 @@ components:
                 "state": null,
                 "city": "Sydney",
                 "asn": 16276,
-                "longitude": 151.207052,
-                "latitude": -33.86778,
+                "longitude": 151.21,
+                "latitude": -33.87,
                 "network": "OVH SAS",
                 "tags": [
                   "datacenter-network"
@@ -220,8 +220,8 @@ components:
                 "state": null,
                 "city": "Sydney",
                 "asn": 202422,
-                "longitude": 151.207052,
-                "latitude": -33.86778,
+                "longitude": 151.21,
+                "latitude": -33.87,
                 "network": "G-Core Labs S.A.",
                 "tags": [
                   "datacenter-network"
@@ -271,8 +271,8 @@ components:
                 "state": null,
                 "city": "Melbourne",
                 "asn": 20473,
-                "longitude": 144.96332,
-                "latitude": -37.814,
+                "longitude": 144.96,
+                "latitude": -37.81,
                 "network": "Choopa, LLC",
                 "tags": [ ],
                 "resolvers": [
@@ -379,8 +379,8 @@ components:
                 "state": null,
                 "city": "Johannesburg",
                 "asn": 199524,
-                "longitude": 28.04355,
-                "latitude": -26.20225,
+                "longitude": 28.04,
+                "latitude": -26.20,
                 "network": "G-Core Labs S.A.",
                 "tags": [
                   "datacenter-network"
@@ -446,8 +446,8 @@ components:
                 "state": null,
                 "city": "Seoul",
                 "asn": 40676,
-                "longitude": 126.977207,
-                "latitude": 37.566309,
+                "longitude": 126.98,
+                "latitude": 37.57,
                 "network": "Psychz Networks",
                 "tags": [
                   "datacenter-network"

src/index.ts

@@ -15,37 +15,50 @@ const workerFn = async () => {
 	const server = await createServer();
 
 	server.listen(port, () => {
-		logger.info(`application started at http://localhost:${port}`);
+		logger.info(`Application started at http://localhost:${port}`);
 	});
 };
 
 if (cluster.isPrimary) {
-	logger.info(`Master ${process.pid} is running with ${workerCount} workers`);
+	logger.info(`Master ${process.pid} is running with ${workerCount} workers.`);
 	const redis = await initRedisClient();
 	const persistentRedis = await initPersistentRedisClient();
 	await flushRedisCache();
 	await redis.disconnect();
 	await persistentRedis.disconnect();
+	let syncAdoptionsPid: number | null = null;
 
 	for (let i = 0; i < workerCount; i++) {
-		cluster.fork();
+		if (!syncAdoptionsPid) {
+			const worker = cluster.fork({ SHOULD_SYNC_ADOPTIONS: true });
+			logger.info(`Syncing adoptions on worker ${worker.process.pid!}.`);
+			syncAdoptionsPid = worker.process.pid!;
+		} else {
+			cluster.fork();
+		}
 	}
 
 	cluster.on('exit', (worker, code, signal) => {
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		logger.error(`worker ${worker.process.pid!} died with code ${code} and signal ${signal}`);
+		logger.error(`Worker ${worker.process.pid!} died with code ${code} and signal ${signal}.`);
 
 		if (process.env['TEST_DONT_RESTART_WORKERS']) {
 			return;
 		}
 
-		cluster.fork();
+		if (worker.process.pid === syncAdoptionsPid) {
+			const worker = cluster.fork({ SHOULD_SYNC_ADOPTIONS: true });
+			logger.info(`Syncing adoptions on worker ${worker.process.pid!}.`);
+			syncAdoptionsPid = worker.process.pid!;
+		} else {
+			cluster.fork();
+		}
 	});
 } else {
-	logger.info(`Worker ${process.pid} is running`);
+	logger.info(`Worker ${process.pid} is running.`);
 
 	workerFn().catch((error) => {
-		logger.error('failed to start cluster', error);
+		logger.error('Failed to start cluster:', error);
 		setTimeout(() => process.exit(1), 5000);
 	});
 }

src/lib/geoip/client.ts

@@ -14,6 +14,7 @@ import { ipmapLookup } from './providers/ipmap.js';
 import { ip2LocationLookup } from './providers/ip2location.js';
 import { isHostingOverrides } from './overrides.js';
 import NullCache from '../cache/null-cache.js';
+import { normalizeCoordinate } from './utils.js';
 
 type Provider = 'ipmap' | 'ip2location' | 'ipinfo' | 'maxmind' | 'fastly';
 export type LocationInfo = ProbeLocation & {isProxy: boolean | null, isHosting: boolean | null, isAnycast: boolean | null};
@@ -104,8 +105,8 @@ export default class GeoIpClient {
 			region: match.region,
 			normalizedCity: match.normalizedCity,
 			asn: Number(networkMatch.asn),
-			latitude: Math.round(Number(match.latitude) * 100) / 100,
-			longitude: Math.round(Number(match.longitude) * 100) / 100,
+			latitude: normalizeCoordinate(Number(match.latitude)),
+			longitude: normalizeCoordinate(Number(match.longitude)),
 			network: networkMatch.network,
 			normalizedNetwork: networkMatch.normalizedNetwork,
 			isProxy,

src/lib/geoip/fake-client.ts

@@ -9,8 +9,8 @@ export const fakeLookup = (): LocationInfo => {
 		region: 'South America',
 		normalizedCity: 'buenos aires',
 		asn: 61003,
-		latitude: -34.6131,
-		longitude: -58.3772,
+		latitude: -34.61,
+		longitude: -58.38,
 		network: 'InterBS S.R.L. (BAEHOST)',
 		normalizedNetwork: 'interbs s.r.l. (baehost)',
 		isProxy: false,

src/lib/geoip/utils.ts

@@ -12,3 +12,5 @@ export const normalizeCityName = (name: string): string => normalizeCityNamePubl
 export const normalizeFromPublicName = (name: string): string => name.toLowerCase();
 
 export const normalizeNetworkName = (name: string): string => name.toLowerCase();
+
+export const normalizeCoordinate = (coordinate: number) => Math.round(coordinate * 100) / 100;

src/lib/get-probe-ip.ts

@@ -13,13 +13,13 @@ const getProbeIp = (socket: Socket) => {
 	// Use random ip assigned by the API
 	if (process.env['FAKE_PROBE_IP']) {
 		const samples = [
+			'131.255.7.26', // Buenos Aires
 			'213.136.174.80', // Naples
+			'95.155.94.127', // Krakow
 			'18.200.0.1', // Dublin, AWS
 			'34.140.0.10', // Brussels, GCP
-			'95.155.94.127', // Krakow
 			'65.49.22.66',
 			'185.229.226.83',
-			'131.255.7.26',
 			'94.214.253.78',
 			'79.205.97.254',
 			'2a04:4e42:200::485', // San Francisco

src/lib/malware/domain.ts

@@ -3,6 +3,9 @@ import path from 'node:path';
 import type { CustomHelpers, ErrorReport } from 'joi';
 import got from 'got';
 import validator from 'validator';
+import { scopedLogger } from '../logger.js';
+
+const logger = scopedLogger('malware-domain');
 
 export const sourceList = [
 	'https://phishing.army/download/phishing_army_blocklist.txt',
@@ -28,7 +31,14 @@ export const query = async (url: string): Promise<string[]> => {
 
 export const updateList = async (): Promise<void> => {
 	const result = await Promise.allSettled(sourceList.map(source => query(source)));
-	const list = [ ...new Set(result.flatMap(r => isFulfilled(r) ? r.value : [])) ].map(d => d.toLowerCase());
+	const list = [ ...new Set(result.flatMap((r) => {
+		if (isFulfilled(r)) {
+			return r.value;
+		}
+
+		logger.error(r.reason);
+		return [];
+	})) ].map(d => d.toLowerCase());
 
 	await writeFile(domainListPath, JSON.stringify(list), { encoding: 'utf8' });
 };

src/lib/malware/ip.ts

@@ -4,6 +4,9 @@ import type { CustomHelpers, ErrorReport } from 'joi';
 import got from 'got';
 import validator from 'validator';
 import ipaddr from 'ipaddr.js';
+import { scopedLogger } from '../logger.js';
+
+const logger = scopedLogger('malware-ip');
 
 export const sourceList = [
 	'https://osint.digitalside.it/Threat-Intel/lists/latestips.txt',
@@ -39,7 +42,14 @@ export const populateMemList = async (): Promise<void> => {
 
 export const updateList = async (): Promise<void> => {
 	const result = await Promise.allSettled(sourceList.map(source => query(source)));
-	const ipList = [ ...new Set(result.flatMap(r => isFulfilled(r) ? r.value : [])) ];
+	const ipList = [ ...new Set(result.flatMap((r) => {
+		if (isFulfilled(r)) {
+			return r.value;
+		}
+
+		logger.error(r.reason);
+		return [];
+	})) ];
 
 	await writeFile(ipListPath, JSON.stringify(ipList), { encoding: 'utf8' });
 };