feat: initially separate on pers and non-pers redis dbs

probes-stats/known.ts

@@ -6,7 +6,7 @@
  */
 
 import fs from 'node:fs';
-import { getRedisClient, initRedis } from '../src/lib/redis/client.js';
+import { getRedisClient, initRedisClient } from '../src/lib/redis/client.js';
 import { LocationInfo, createGeoipClient } from '../src/lib/geoip/client.js';
 import { normalizeCityName } from '../src/lib/geoip/utils.js';
 import { fastlyLookup } from '../src/lib/geoip/providers/fastly.js';
@@ -19,7 +19,7 @@ import sheet from './known-probes.json' assert { type: 'json' };
 import { populateCitiesList } from '../src/lib/geoip/city-approximation.js';
 
 await populateIpWhiteList();
-await initRedis();
+await initRedisClient();
 await populateCitiesList();
 const geoIpClient = createGeoipClient();
 

src/lib/metrics.ts

@@ -2,9 +2,9 @@ import process from 'node:process';
 import type { Server as SocketServer } from 'socket.io';
 import newrelic from 'newrelic';
 
-import { getRedisClient, type RedisClient } from './redis/client.js';
 import { getWsServer, PROBES_NAMESPACE } from './ws/server.js';
 import { scopedLogger } from './logger.js';
+import { getPersistentRedisClient, PersistentRedisClient } from './redis/persistent-client.js';
 
 const logger = scopedLogger('metrics');
 
@@ -13,7 +13,7 @@ export class MetricsAgent {
 
 	constructor (
 		private readonly io: SocketServer,
-		private readonly redis: RedisClient,
+		private readonly redis: PersistentRedisClient,
 	) {}
 
 	run (): void {
@@ -62,7 +62,7 @@ let agent: MetricsAgent;
 
 export const getMetricsAgent = () => {
 	if (!agent) {
-		agent = new MetricsAgent(getWsServer(), getRedisClient());
+		agent = new MetricsAgent(getWsServer(), getPersistentRedisClient());
 	}
 
 	return agent;

src/lib/ratelimiter.ts

@@ -3,10 +3,10 @@ import type { Context } from 'koa';
 import { RateLimiterRedis } from 'rate-limiter-flexible';
 import requestIp from 'request-ip';
 import type { RateLimiterRes } from 'rate-limiter-flexible';
-import { createRedisClient } from './redis/client.js';
+import { createPersistentRedisClient } from './redis/persistent-client.js';
 import createHttpError from 'http-errors';
 
-const redisClient = await createRedisClient({ legacyMode: true });
+const redisClient = await createPersistentRedisClient({ legacyMode: true });
 
 const rateLimiter = new RateLimiterRedis({
 	storeClient: redisClient,

src/lib/redis/client.ts

@@ -15,14 +15,17 @@ export type RedisClient = RedisClientType<RedisDefaultModules, RedisFunctions, R
 
 let redis: RedisClient;
 
-export const initRedis = async () => {
+export const initRedisClient = async () => {
 	redis = await createRedisClient();
+	await redis.flushDb();
 };
 
-export const createRedisClient = async (options?: RedisClientOptions): Promise<RedisClient> => {
+const createRedisClient = async (options?: RedisClientOptions): Promise<RedisClient> => {
 	const client = createClient({
 		...config.util.toObject(config.get('redis')) as RedisClientOptions,
 		...options,
+		database: 0,
+		name: 'non-persistent',
 		scripts: { count, recordResult, markFinished },
 	});
 
@@ -38,7 +41,7 @@ export const createRedisClient = async (options?: RedisClientOptions): Promise<R
 
 export const getRedisClient = (): RedisClient => {
 	if (!redis) {
-		throw new Error('redis connection is not initialize yet');
+		throw new Error('redis connection is not initialized yet');
 	}
 
 	return redis;

src/lib/redis/persistent-client.ts

@@ -0,0 +1,47 @@
+import config from 'config';
+import {
+	createClient,
+	type RedisClientType,
+	type RedisDefaultModules,
+	type RedisClientOptions,
+	type RedisFunctions,
+} from 'redis';
+import { scopedLogger } from '../logger.js';
+import { count, recordResult, markFinished, type RedisScripts } from './scripts.js';
+
+const logger = scopedLogger('persistent-redis-client');
+
+export type PersistentRedisClient = RedisClientType<RedisDefaultModules, RedisFunctions, RedisScripts>;
+
+let redis: PersistentRedisClient;
+
+export const initPersistentRedisClient = async () => {
+	redis = await createPersistentRedisClient();
+};
+
+export const createPersistentRedisClient = async (options?: RedisClientOptions): Promise<PersistentRedisClient> => {
+	const client = createClient({
+		...config.util.toObject(config.get('redis')) as RedisClientOptions,
+		...options,
+		database: 1,
+		name: 'persistent',
+		scripts: { count, recordResult, markFinished },
+	});
+
+	client
+		.on('error', (error: Error) => logger.error('connection error', error))
+		.on('ready', () => logger.info('connection ready'))
+		.on('reconnecting', () => logger.info('reconnecting'));
+
+	await client.connect();
+
+	return client;
+};
+
+export const getPersistentRedisClient = (): PersistentRedisClient => {
+	if (!redis) {
+		throw new Error('redis connection to persistent db is not initialized yet');
+	}
+
+	return redis;
+};

src/lib/server.ts

@@ -1,5 +1,5 @@
 import type { Server } from 'node:http';
-import { initRedis } from './redis/client.js';
+import { initRedisClient } from './redis/client.js';
 import { initWsServer } from './ws/server.js';
 import { getMetricsAgent } from './metrics.js';
 import { populateMemList as populateMemMalwareList } from './malware/client.js';
@@ -8,9 +8,11 @@ import { populateMemList as populateIpWhiteList } from './geoip/whitelist.js';
 import { populateCitiesList } from './geoip/city-approximation.js';
 import { adoptedProbes } from './adopted-probes.js';
 import { reconnectProbes } from './ws/helper/reconnect-probes.js';
+import { initPersistentRedisClient } from './redis/persistent-client.js';
 
 export const createServer = async (): Promise<Server> => {
-	await initRedis();
+	await initRedisClient();
+	await initPersistentRedisClient();
 
 	// Populate memory malware list
 	await populateMemMalwareList();

src/measurement/store.ts

@@ -2,11 +2,10 @@ import config from 'config';
 import _ from 'lodash';
 import cryptoRandomString from 'crypto-random-string';
 import type { OfflineProbe, Probe } from '../probe/types.js';
-import type { RedisClient } from '../lib/redis/client.js';
-import { getRedisClient } from '../lib/redis/client.js';
 import { scopedLogger } from '../lib/logger.js';
 import type { MeasurementRecord, MeasurementResult, MeasurementRequest, MeasurementProgressMessage, RequestType, MeasurementResultMessage } from './types.js';
 import { getDefaults } from './schema/utils.js';
+import { getPersistentRedisClient, PersistentRedisClient } from '../lib/redis/persistent-client.js';
 
 const logger = scopedLogger('store');
 
@@ -42,7 +41,7 @@ const substractObjects = (obj1: Record<string, unknown>, obj2: Record<string, un
 };
 
 export class MeasurementStore {
-	constructor (private readonly redis: RedisClient) {}
+	constructor (private readonly redis: PersistentRedisClient) {}
 
 	async getMeasurementString (id: string): Promise<string> {
 		return this.redis.sendCommand([ 'JSON.GET', getMeasurementKey(id) ]);
@@ -239,7 +238,7 @@ let store: MeasurementStore;
 
 export const getMeasurementStore = () => {
 	if (!store) {
-		store = new MeasurementStore(getRedisClient());
+		store = new MeasurementStore(getPersistentRedisClient());
 		store.scheduleCleanup();
 	}
 

test/setup.ts

@@ -14,7 +14,8 @@ import {
 	populateNockCitiesList,
 } from './utils/populate-static-files.js';
 import chaiOas from './plugins/oas/index.js';
-import { getRedisClient, initRedis } from '../src/lib/redis/client.js';
+import { initRedisClient } from '../src/lib/redis/client.js';
+import { getPersistentRedisClient, initPersistentRedisClient } from '../src/lib/redis/persistent-client.js';
 import { client as sql } from '../src/lib/sql/client.js';
 
 const dbConfig = config.get<{ connection: { database: string, host: string } }>('db');
@@ -26,9 +27,10 @@ if (!dbConfig.connection.database.endsWith('-test') && dbConfig.connection.host
 before(async () => {
 	chai.use(await chaiOas({ specPath: path.join(fileURLToPath(new URL('.', import.meta.url)), '../public/v1/spec.yaml') }));
 
-	await initRedis();
-	const redis = getRedisClient();
-	await redis.flushDb();
+	await initRedisClient();
+	await initPersistentRedisClient();
+	const persistentRedisClient = getPersistentRedisClient();
+	await persistentRedisClient.flushDb();
 
 	await dropAllTables(sql);
 	await sql.migrate.latest();

test/tests/unit/measurement/store.test.ts

@@ -48,7 +48,7 @@ describe('measurement store', () => {
 
 	before(async () => {
 		td.replaceEsm('crypto-random-string', null, () => 'measurementid');
-		await td.replaceEsm('../../../../src/lib/redis/client.ts', { getRedisClient: () => redisMock });
+		await td.replaceEsm('../../../../src/lib/redis/persistent-client.ts', { getPersistentRedisClient: () => redisMock });
 		getMeasurementStore = (await import('../../../../src/measurement/store.js')).getMeasurementStore;
 	});