feat(access-logs): replace DuckDB/Parquet with external ClickHouse backend

- Panel is now a thin gateway: raw lines go to ClickHouse, parsing via versioned materialized view
- Fix regex escaping in MV DDL, date normalization, UTC handling end-to-end
- Batch retry dedup via insert_deduplication_token + dependent MV dedup
- Remove all legacy DuckDB/Parquet/rollup code, dead JS parser and unused locale keys

Author: ddddeadispose

.gitignore

@@ -47,7 +47,7 @@ cc-agent/cc-agent
 cc-agent/cc-agent-linux-amd64
 cc-agent/cc-agent-linux-arm64
 
-# Access-logs pipeline data (Parquet store + ingest spool)
+# Access-logs ingest spool
 data/access-logs/
 
 Rules.md

Dockerfile

@@ -3,9 +3,10 @@ FROM node:20-alpine
 
 WORKDIR /app
 
-# System dependencies: mongodb-tools for backups; libstdc++/libgcc are required
-# by the DuckDB native binding used by the access-logs analytics pipeline.
-RUN apk add --no-cache mongodb-tools libstdc++ libgcc
+# System dependencies: mongodb-tools for backups. The access-logs pipeline now
+# talks to an external ClickHouse over HTTP (pure-JS client), so no native
+# runtime libraries are needed.
+RUN apk add --no-cache mongodb-tools
 
 # Копируем зависимости
 COPY package*.json ./

docker-compose.dokploy.yml

@@ -52,7 +52,7 @@ services:
     volumes:
       - app_logs:/app/logs
       - app_backups:/app/backups
-      # Access-logs pipeline data (Parquet store + ingest spool).
+      # Access-logs ingest spool (batches not yet forwarded to ClickHouse).
       - app_data:/app/data
 
 volumes:

docker-compose.hub.yml

@@ -55,7 +55,7 @@ services:
       - ./logs:/app/logs
       - ./greenlock.d:/app/greenlock.d
       - ./backups:/app/backups
-      # Access-logs pipeline data (Parquet store + ingest spool) must survive
+      # Access-logs ingest spool (batches not yet forwarded to ClickHouse) must survive
       # container rebuilds.
       - ./data:/app/data
     env_file:

docker-compose.local.yml

@@ -59,7 +59,7 @@ services:
     volumes:
       - ./logs:/app/logs
       - ./backups:/app/backups
-      # Access-logs pipeline data (Parquet store + ingest spool).
+      # Access-logs ingest spool (batches not yet forwarded to ClickHouse).
       - ./data:/app/data
     environment:
       MONGO_URI: mongodb://${MONGO_USER:-hysteria}:${MONGO_PASSWORD:-localdevpassword}@mongo:27017/hysteria?authSource=admin

docker-compose.yml

@@ -72,8 +72,8 @@ services:
     volumes:
       - ./logs:/app/logs
       - ./backups:/app/backups
-      # Access-logs pipeline data (Parquet store + ingest spool) must survive
-      # container rebuilds.
+      # Access-logs ingest spool must survive container rebuilds (batches not
+      # yet forwarded to ClickHouse live here).
       - ./data:/app/data
       - caddy_data:/caddy_data:ro
     env_file:

index.js

@@ -714,14 +714,29 @@ async function startServer() {
         setupCronJobs();
 
         // Access-logs spool processor. Always started: it is idle when the spool
-        // is empty (feature off) and picks up batches immediately once an admin
-        // enables collection, without needing a restart.
+        // is empty (feature off / ClickHouse not configured) and picks up batches
+        // immediately once collection is enabled, without needing a restart.
         try {
             require('./src/services/accessLogs/processService').start();
         } catch (e) {
             logger.warn(`[AccessLogs] processor start skipped: ${e.message}`);
         }
 
+        // Ensure the ClickHouse schema exists on boot when credentials are set,
+        // so the pipeline can insert immediately. Idempotent (IF NOT EXISTS) and
+        // best-effort: a missing/unreachable ClickHouse just logs and retries on
+        // the next settings save.
+        setTimeout(async () => {
+            try {
+                const clickhouse = require('./src/services/accessLogs/clickhouseService');
+                if (await clickhouse.isConfigured()) {
+                    await clickhouse.ensureSchema();
+                }
+            } catch (e) {
+                logger.warn(`[AccessLogs] ClickHouse schema ensure at boot skipped: ${e.message}`);
+            }
+        }, 15 * 1000);
+
         // Crash recovery: if the panel died mid-reconcile the access-logs state
         // stays stuck in a transitional value. Re-run reconciliation once on
         // boot (delayed so nodes/DB settle first); it is a no-op fast path when
@@ -1031,19 +1046,8 @@ function setupCronJobs() {
         }
     });
 
-    // Access-logs retention & storage-cap enforcement (daily 03:20). Cheap no-op
-    // when the feature is off or the store is empty.
-    cron.schedule('20 3 * * *', async () => {
-        try {
-            const Settings = require('./src/models/settingsModel');
-            const settings = await Settings.get();
-            if (settings?.accessLogs?.enabled) {
-                await require('./src/services/accessLogs/retentionService').enforce();
-            }
-        } catch (error) {
-            logger.error(`[Cron] Access-logs retention failed: ${error.message}`);
-        }
-    });
+    // Access-logs retention is enforced natively by the ClickHouse TTL on the
+    // access_events table (set via settings), so the panel runs no retention job.
 
     // Clean inactive HWID device rows (daily 03:30)
     cron.schedule('30 3 * * *', async () => {

package-lock.json

@@ -33,12 +33,12 @@
     "start": "node index.js",
     "dev": "nodemon index.js",
     "test": "node scripts/test-session-cookie-config.js && node scripts/test-i18n-fallback.js && node scripts/test-node-ui-meta.js && node scripts/test-node-active-api.js && node scripts/test-auth-subscription-boundaries.js && node scripts/test-setup-ejs-validation.js && node scripts/test-security-ejs-validation.js && node scripts/test-remote-cron-service.js && node scripts/test-node-cron-panel.js && node scripts/test-node-cron-ui.js && node scripts/test-multicast-cron-service.js && node scripts/test-broadcast-panel.js && node scripts/test-broadcast-ui.js && npm run test:access-logs",
-    "test:access-logs": "node scripts/test-access-logs-contract.js && node scripts/test-access-logs-credentials.js && node scripts/test-access-logs-ingest.js && node scripts/test-access-logs-parquet.js && node scripts/test-access-logs-retention.js",
+    "test:access-logs": "node scripts/test-access-logs-credentials.js && node scripts/test-access-logs-ingest.js && node scripts/test-access-logs-clickhouse.js",
     "build:assets": "npx --yes esbuild public/css/style.css public/css/network.css --minify --loader:.css=css --outdir=public/css --out-extension:.css=.min.css --allow-overwrite && npx --yes esbuild public/js/network.js --minify --outfile=public/js/network.min.js"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.500.0",
-    "@duckdb/node-api": "^1.5.4-r.1",
+    "@clickhouse/client": "^1.23.1",
     "@root/greenlock-express": "^4.0.3",
     "axios": "^1.6.2",
     "bcryptjs": "^2.4.3",

package.json

@@ -1,11 +1,10 @@
 // Access-logs dashboard: filters, analytics overview (charts + user/top tables),
 // on-demand event search, node status, purge.
 //
-// All DuckDB-backed data comes from a single /api/analytics call (the server
-// runs the whole overview in one worker spawn). The raw-event search is a
-// separate on-demand request. Requests are issued sequentially, never in
-// parallel, because the panel intentionally allows only one heavy DuckDB query
-// at a time (weak-hardware constraint) and parallel calls would be rejected.
+// The whole overview comes from a single /api/analytics call (the server fans
+// the aggregate queries out to ClickHouse). The raw-event search is a separate
+// on-demand request. Requests are issued sequentially so the slower analytics
+// response does not delay the search rows the user is usually after.
 (function () {
     'use strict';
 
@@ -43,8 +42,8 @@
         return n.toFixed(i ? 1 : 0) + ' ' + u[i];
     }
 
-    // DuckDB returns naive UTC timestamps ("2026-07-10 10:05:00", no zone). Parse
-    // them as UTC so the whole page displays in the viewer's LOCAL time —
+    // The server returns naive UTC timestamps ("2026-07-10 10:05:00", no zone).
+    // Parse them as UTC so the whole page displays in the viewer's LOCAL time —
     // consistent with the datetime-local filter inputs (which are local and get
     // converted to UTC for the query). Without this the shown times would be
     // silently shifted by the timezone offset.
@@ -359,9 +358,10 @@
             renderUsers(data.users);
             renderTops(data);
 
-            // In degraded mode the DuckDB-only widgets are empty; hint why.
-            if (data.duckdbRequired) {
-                const note = (span) => '<tr><td class="al-empty" colspan="' + span + '">' + esc(I18N.duckdbRequired) + '</td></tr>';
+            // Degraded mode (ClickHouse not configured/unreachable): widgets are
+            // empty; hint why.
+            if (data.chRequired) {
+                const note = (span) => '<tr><td class="al-empty" colspan="' + span + '">' + esc(I18N.chRequired) + '</td></tr>';
                 if (!(data.users || []).length) { $('alUsersByIp').innerHTML = note(5); $('alUsersByFanout').innerHTML = note(4); }
                 if (!(data.topPorts || []).length) $('alTopPorts').innerHTML = note(2);
                 if (!(data.topBlocked || []).length) $('alTopBlocked').innerHTML = note(2);
@@ -417,8 +417,8 @@
         }
     }
 
-    // Sequential refresh: analytics -> search -> status (never parallel, to
-    // respect the single-concurrent-DuckDB-query limit).
+    // Sequential refresh: analytics -> search -> status. Keeps the page calm and
+    // avoids piling concurrent aggregate queries onto ClickHouse from one tab.
     async function refreshAll() {
         await loadAnalytics();
         await loadSearch();