perf: use a redis cluster

.dockerignore

@@ -5,3 +5,4 @@ coverage/
 dist/
 node_modules/
 probes-stats/
+test/

.github/workflows/ci.yml

@@ -15,16 +15,6 @@ jobs:
       NODE_ENV: test
 
     services:
-      redis:
-        image: redis/redis-stack-server:latest
-        ports:
-          - 16379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
       mariadb:
         image: mariadb:10.11.5
         ports:
@@ -45,6 +35,10 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20.x
+      - name: Set up Redis
+        run: |
+          cp config/redis/.env.redis ./
+          docker compose --env-file .env.redis up -d
       - name: Build
         run: |
           npm ci

.github/workflows/e2e.yml

@@ -15,16 +15,6 @@ jobs:
       NODE_ENV: test
 
     services:
-      redis:
-        image: redis/redis-stack-server:latest
-        ports:
-          - 16379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
       mariadb:
         image: mariadb:10.11.5
         ports:
@@ -51,6 +41,10 @@ jobs:
           sudo mv daemon.json /etc/docker/
           sudo systemctl restart docker
           docker restart $(docker ps -aq)
+      - name: Set up Redis
+        run: |
+          cp config/redis/.env.redis ./
+          docker compose --env-file .env.redis up -d
       - name: Build
         run: |
           npm ci

.gitignore

@@ -6,6 +6,7 @@ coverage/
 dist/
 tmp/
 config/local*
+data/redis
 data/DOMAIN_BLACKLIST.json
 data/IP_BLACKLIST.json
 data/AWS_IP_RANGES.json
@@ -18,3 +19,4 @@ probes-stats/all-result.csv
 probes-stats/all-result.json
 .eslintcache
 .env
+/.env.redis

.husky/pre-commit

@@ -6,4 +6,4 @@ if ! type cygpath > /dev/null 2>&1; then
 	PATH="$PATH:$(type -ap git | grep 'cmd/git' | sed 's$cmd/git$usr/bin$')"
 fi
 
-node_modules/.bin/lint-staged --no-stash --quiet
+node_modules/.bin/lint-staged --quiet

CONTRIBUTING.md

@@ -48,3 +48,4 @@ Most IDEs have plugins integrating the used linter (eslint), including support f
 - `SYSTEM_API_KEY={value}` used for integration with the dashboard
 - `SERVER_SESSION_COOKIE_SECRET={value}` used to read the shared session cookie
 - `DB_CONNECTION_HOST`, `DB_CONNECTION_USER`, `DB_CONNECTION_PASSWORD`, and `DB_CONNECTION_DATABASE` database connection details
+- `REDIS_STANDALONE_PERSISTENT_URL`, `REDIS_STANDALONE_NON_PERSISTENT_URL`, `REDIS_CLUSTER_MEASUREMENTS_NODES_0`, `REDIS_CLUSTER_MEASUREMENTS_NODES_1`, `REDIS_CLUSTER_MEASUREMENTS_NODES_2`, and `REDIS_SHARED_OPTIONS_PASSWORD` - redis connection details

Dockerfile

@@ -1,7 +1,7 @@
 FROM node:20-bullseye-slim AS builder
 RUN apt-get update -y && apt-get install util-linux curl git -y
 
-ENV NODE_ENV production
+ENV NODE_ENV=production
 
 COPY package.json package-lock.json /app/
 WORKDIR /app
@@ -12,7 +12,7 @@ RUN npm run build
 FROM node:20-bullseye-slim
 RUN apt-get update -y && apt-get install tini util-linux curl -y
 
-ENV NODE_ENV production
+ENV NODE_ENV=production
 
 COPY package.json package-lock.json /app/
 WORKDIR /app

config/custom-environment-variables.cjs

@@ -1,25 +1,2 @@
-const _ = require('lodash');
 const df = require('./default.cjs');
-
-function mapEnvConfig (object, prefix = '') {
-	return _.mapValues(object, (value, key) => {
-		const currentKey = (prefix ? `${prefix}_` : '') + _.snakeCase(key).toUpperCase();
-
-		if (_.isObject(value)) {
-			return mapEnvConfig(value, currentKey);
-		}
-
-		if (typeof value === 'number' || typeof value === 'boolean') {
-			return {
-				__name: currentKey,
-				__format: typeof value,
-			};
-		}
-
-		return currentKey;
-	});
-}
-
-const mapped = mapEnvConfig(df);
-
-module.exports = mapped;
+module.exports = require('config-mapper-env')(df);

config/default.cjs

@@ -17,9 +17,26 @@ module.exports = {
 		},
 	},
 	redis: {
-		url: 'redis://localhost:6379',
-		socket: {
-			tls: false,
+		standalonePersistent: {
+			url: 'redis://localhost:7001',
+		},
+		standaloneNonPersistent: {
+			url: 'redis://localhost:7002',
+		},
+		clusterMeasurements: {
+			// listing three nodes here is enough, the rest will be discovered automatically
+			nodes: {
+				0: 'redis://localhost:7101',
+				1: 'redis://localhost:7102',
+				2: 'redis://localhost:7103',
+			},
+			options: {},
+		},
+		sharedOptions: {
+			password: 'PASSWORD',
+			socket: {
+				tls: false,
+			},
 		},
 	},
 	db: {

config/development.cjs

@@ -4,12 +4,6 @@ module.exports = {
 			cookieSecret: 'xxx',
 		},
 	},
-	redis: {
-		url: 'redis://localhost:16379',
-		socket: {
-			tls: false,
-		},
-	},
 	db: {
 		connection: {
 			port: 13306,

config/redis/.env.redis

@@ -0,0 +1,6 @@
+# standalone:
+REDIS_ARGS="--requirepass PASSWORD"
+
+# cluster:
+REDIS_PASSWORD=PASSWORD
+REDIS_PUBLIC_IP=127.0.0.1

redis/setup.md → config/redis/README.md

@@ -12,9 +12,16 @@ swapon /swapfile
 echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
 ```
 
-## Config
+### Docker config
 
-1. Download redis.conf to /etc/redis/
-2. Download zip file with json module to same folder
-3. Unzip
-4. Restart
+Assuming you start in this directory:
+
+```
+cp .env.redis ../../
+```
+
+Set the redis password and return to the project root. Then:
+
+```
+docker compose --env-file .env.redis up -d
+```