Report stats about blocked requests

end2end/server/src/handlers/lists.js

@@ -26,7 +26,9 @@ module.exports = function lists(req, res) {
             },
           ]
         : [],
-    blockedUserAgents: blockedUserAgents,
+    blockedUserAgentsV2: Array.isArray(blockedUserAgents)
+      ? blockedUserAgents
+      : [],
     allowedIPAddresses:
       allowedIps.length > 0
         ? [

end2end/server/src/handlers/updateLists.js

@@ -33,10 +33,10 @@ module.exports = function updateIPLists(req, res) {
   updateBlockedIPAddresses(req.app, req.body.blockedIPAddresses);
 
   if (
-    req.body.blockedUserAgents &&
-    typeof req.body.blockedUserAgents === "string"
+    req.body.blockedUserAgentsV2 &&
+    Array.isArray(req.body.blockedUserAgentsV2)
   ) {
-    updateBlockedUserAgents(req.app, req.body.blockedUserAgents);
+    updateBlockedUserAgents(req.app, req.body.blockedUserAgentsV2);
   }
 
   if (

end2end/tests/hono-xml-blocklists.test.js

@@ -45,7 +45,12 @@ t.beforeEach(async () => {
     },
     body: JSON.stringify({
       blockedIPAddresses: ["1.3.2.0/24", "e98c:a7ba:2329:8c69::/64"],
-      blockedUserAgents: "hacker|attacker|GPTBot",
+      blockedUserAgentsV2: [
+        {
+          key: "some/key",
+          pattern: "hacker|attacker|GPTBot",
+        },
+      ],
     }),
   });
   t.same(lists.status, 200);

library/agent/Agent.test.ts

@@ -18,6 +18,7 @@ import { Wrapper } from "./Wrapper";
 import { Context } from "./Context";
 import { createTestAgent } from "../helpers/createTestAgent";
 import { setTimeout } from "node:timers/promises";
+import type { Response } from "./api/fetchBlockedLists";
 
 let shouldOnlyAllowSomeIPAddresses = false;
 
@@ -28,22 +29,33 @@ wrap(fetch, "fetch", function mock() {
       body: JSON.stringify({
         blockedIPAddresses: [
           {
+            key: "some/key",
             source: "name",
             description: "Description",
             ips: ["1.3.2.0/24", "fe80::1234:5678:abcd:ef12/64"],
           },
         ],
-        blockedUserAgents: "AI2Bot|Bytespider",
+        blockedUserAgentsV2: [
+          {
+            key: "ai",
+            pattern: "AI2Bot|SomethingElse",
+          },
+          {
+            key: "spider",
+            pattern: "Bytespider",
+          },
+        ],
         allowedIPAddresses: shouldOnlyAllowSomeIPAddresses
           ? [
               {
+                key: "some/key",
                 source: "name",
                 description: "Description",
                 ips: ["4.3.2.1"],
               },
             ]
           : [],
-      }),
+      } satisfies Response),
     };
   };
 });
@@ -1066,10 +1078,12 @@ t.test("it fetches blocked lists", async () => {
   t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
     blocked: true,
     reason: "Description",
+    key: "some/key",
   });
   t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
     blocked: true,
     reason: "Description",
+    key: "some/key",
   });
 
   t.same(
@@ -1079,13 +1093,15 @@ t.test("it fetches blocked lists", async () => {
         "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)"
       ),
     {
+      key: "ai",
       blocked: true,
     }
   );
 
   t.same(
     agent.getConfig().isUserAgentBlocked("Mozilla/5.0 (compatible) Bytespider"),
     {
+      key: "spider",
       blocked: true,
     }
   );
@@ -1140,10 +1156,12 @@ t.test("it only allows some IP addresses", async () => {
   t.same(agent.getConfig().isIPAddressBlocked("1.3.2.4"), {
     blocked: true,
     reason: "Description",
+    key: "some/key",
   });
   t.same(agent.getConfig().isIPAddressBlocked("fe80::1234:5678:abcd:ef12"), {
     blocked: true,
     reason: "Description",
+    key: "some/key",
   });
 
   t.same(agent.getConfig().isAllowedIPAddress("1.2.3.4"), {

library/agent/InspectionStatistics.test.ts

@@ -39,6 +39,11 @@ t.test("it resets stats", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -54,6 +59,11 @@ t.test("it resets stats", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -80,6 +90,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -112,6 +127,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -144,6 +164,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -170,6 +195,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -202,6 +232,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -234,6 +269,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -285,6 +325,11 @@ t.test("it keeps track of amount of calls", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -332,6 +377,11 @@ t.test("it keeps track of requests", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -347,6 +397,11 @@ t.test("it keeps track of requests", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -363,6 +418,11 @@ t.test("it keeps track of requests", async () => {
         total: 1,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -379,6 +439,11 @@ t.test("it keeps track of requests", async () => {
         total: 2,
         blocked: 1,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -396,6 +461,11 @@ t.test("it keeps track of requests", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -420,6 +490,11 @@ t.test("it force compresses stats", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
 
@@ -462,6 +537,54 @@ t.test("it keeps track of aborted requests", async () => {
         total: 0,
         blocked: 0,
       },
+      blocked: {
+        total: 0,
+        userAgentList: {},
+        ipBlocklist: {},
+      },
     },
   });
+
+  clock.uninstall();
+});
+
+t.test("it keeps track of blocked requests", async () => {
+  const clock = FakeTimers.install();
+
+  const stats = new InspectionStatistics({
+    maxPerfSamplesInMemory: 50,
+    maxCompressedStatsInMemory: 5,
+  });
+
+  stats.onBlockedRequest({
+    match: "ipBlocklist",
+    key: "known_threat_actors/public_scanners",
+  });
+  stats.onBlockedRequest({ match: "userAgentList", key: "ai_data_scrapers" });
+
+  t.same(stats.getStats(), {
+    sinks: {},
+    startedAt: 0,
+    requests: {
+      total: 0,
+      aborted: 0,
+      attacksDetected: {
+        total: 0,
+        blocked: 0,
+      },
+      blocked: {
+        total: 2,
+        ipBlocklist: {
+          // eslint-disable-next-line camelcase
+          "known_threat_actors/public_scanners": 1,
+        },
+        userAgentList: {
+          // eslint-disable-next-line camelcase
+          ai_data_scrapers: 1,
+        },
+      },
+    },
+  });
+
+  clock.uninstall();
 });