Add option to shard attachments integrity check (#1521)

* Fix tests after #1517

* Add option to shard attachments integrity check

* Secrets baseline

.secrets.baseline

@@ -142,14 +142,14 @@
         "filename": "tests/checks/remotesettings/test_attachments_integrity.py",
         "hashed_secret": "263e1869fb57deeb75844cf85d55f0d03a6019fc",
         "is_verified": false,
-        "line_number": 32
+        "line_number": 36
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/checks/remotesettings/test_attachments_integrity.py",
         "hashed_secret": "fdf34abe05071170ce2ec082a18aa762ec454ae3",
         "is_verified": false,
-        "line_number": 40
+        "line_number": 44
       }
     ],
     "tests/checks/remotesettings/test_public_suffix_list.py": [
@@ -162,5 +162,5 @@
       }
     ]
   },
-  "generated_at": "2024-12-03T11:11:17Z"
+  "generated_at": "2024-12-12T11:31:49Z"
 }

checks/remotesettings/attachments_integrity.py

@@ -5,6 +5,7 @@
 """
 
 import hashlib
+import math
 
 import aiohttp
 
@@ -32,7 +33,7 @@ async def test_attachment(session, attachment):
     return {}, True
 
 
-async def run(server: str) -> CheckResult:
+async def run(server: str, slice_percent: tuple[int, int] = (0, 100)) -> CheckResult:
     client = KintoClient(server_url=server)
 
     info = await client.server_info()
@@ -61,8 +62,14 @@ async def run(server: str) -> CheckResult:
             attachment["location"] = base_url + attachment["location"]
             attachments.append(attachment)
 
+    lower_idx = math.floor(slice_percent[0] / 100.0 * len(attachments))
+    upper_idx = math.ceil(slice_percent[1] / 100.0 * len(attachments))
+
     async with ClientSession() as session:
-        futures = [test_attachment(session, attachment) for attachment in attachments]
+        futures = [
+            test_attachment(session, attachment)
+            for attachment in attachments[lower_idx:upper_idx]
+        ]
         results = await run_parallel(*futures)
     bad = [result for result, success in results if not success]
     return len(bad) == 0, {"bad": bad, "checked": len(attachments)}

tests/checks/remotesettings/test_attachments_integrity.py

@@ -1,3 +1,7 @@
+from unittest import mock
+
+import pytest
+
 from checks.remotesettings.attachments_integrity import run
 
 
@@ -114,3 +118,54 @@ async def test_negative(mock_responses, mock_aioresponses):
         ],
         "checked": 3,
     }
+
+
+@pytest.mark.parametrize(
+    ("slice_percent", "expected_lower", "expected_upper"),
+    [
+        ((0, 100), 0, 99),
+        ((0, 25), 0, 24),
+        ((25, 50), 25, 49),
+        ((50, 75), 50, 74),
+        ((75, 100), 75, 99),
+        ((0, 33), 0, 32),
+        ((33, 66), 33, 65),
+        ((66, 100), 66, 99),
+    ],
+)
+async def test_urls_slicing(
+    slice_percent, expected_lower, expected_upper, mock_responses
+):
+    server_url = "http://fake.local/v1"
+    mock_responses.get(
+        server_url + "/",
+        payload={"capabilities": {"attachments": {"base_url": "http://cdn/"}}},
+    )
+    changes_url = server_url + CHANGESET_URL.format("monitor", "changes")
+    mock_responses.get(
+        changes_url,
+        payload={
+            "changes": [
+                {"id": "abc", "bucket": "bid", "collection": "cid", "last_modified": 42}
+            ]
+        },
+    )
+    records_url = server_url + RECORDS_URL.format("bid", "cid") + "?_expected=42"
+    mock_responses.get(
+        records_url,
+        payload={
+            "data": [
+                {"id": f"id{i}", "attachment": {"location": f"file{i}.jpg"}}
+                for i in range(100)
+            ]
+        },
+    )
+
+    with mock.patch(
+        "checks.remotesettings.attachments_integrity.test_attachment"
+    ) as mocked:
+        mocked.return_value = {}, True
+        await run(server_url, slice_percent=slice_percent)
+    calls = mocked.call_args_list
+    assert calls[0][0][1]["location"] == f"http://cdn/file{expected_lower}.jpg"
+    assert calls[-1][0][1]["location"] == f"http://cdn/file{expected_upper}.jpg"