tests: add deduplication tests for buzhash64(e)

this will detect if there is anything going wrong
regarding deduplication with the encrypted buzhash mode.

Author: ThomasWaldmann

src/borg/testsuite/chunkers/buzhash64_test.py

@@ -2,6 +2,8 @@
 from io import BytesIO
 import os
 
+import pytest
+
 from . import cf
 from ...chunkers import ChunkerBuzHash64
 from ...chunkers.buzhash64 import buzhash64_get_table
@@ -98,3 +100,43 @@ def test_buzhash64_table():
     for bit_pos in range(64):
         bit_count = sum(1 for value in table0 if value & (1 << bit_pos))
         assert bit_count == 128  # 50% of 256 = 128
+
+
+@pytest.mark.parametrize("do_encrypt", (False, True))
+def test_buzhash64_dedup_shifted(do_encrypt):
+    min_exp, max_exp, mask = 10, 16, 14  # chunk size target 16kiB, clip at 1kiB and 64kiB
+    key = b"0123456789ABCDEF" * 2
+    chunker = ChunkerBuzHash64(key, min_exp, max_exp, mask, 4095, do_encrypt=do_encrypt)
+    rdata = os.urandom(4000000)
+
+    def chunkit(data):
+        size = 0
+        chunks = []
+        with BytesIO(data) as f:
+            for chunk in chunker.chunkify(f):
+                chunks.append(sha256(chunk.data).digest())
+                size += len(chunk.data)
+        return chunks, size
+
+    # 2 identical files
+    data1, data2 = rdata, rdata
+    chunks1, size1 = chunkit(data1)
+    chunks2, size2 = chunkit(data2)
+    # exact same chunking
+    assert size1 == len(data1)
+    assert size2 == len(data2)
+    assert chunks1 == chunks2
+
+    # 2 almost identical files
+    data1, data2 = rdata, b"inserted" + rdata
+    chunks1, size1 = chunkit(data1)
+    chunks2, size2 = chunkit(data2)
+    assert size1 == len(data1)
+    assert size2 == len(data2)
+    # almost same chunking
+    # many chunks overall
+    assert len(chunks1) > 100
+    assert len(chunks2) > 100
+    # only a few unique chunks per file, most chunks are duplicates
+    assert len(set(chunks1) - set(chunks2)) <= 2
+    assert len(set(chunks2) - set(chunks1)) <= 2