blake2b_256: fix key passing, fixes #8867

borg 1.x involved the key a bit unusually in the blake2b
hash calculation (padding 64B of key material with another
64B of zeros to a total of 128B and then just prepending
it to the data).

since a while, we use blake2b from python stdlib and they
support passing the key as a separate argument. up to 64B
keys are allowed here.

besides the way how the 64B of key material are passed, there
are also other differences in how the key gets involved in
the digest computation, thus the legacy blake2b hashes from
borg 1.x are not compatible with the non-legacy ones - the
hash is not the same for same key and data values.

Author: ThomasWaldmann

src/borg/archiver/benchmark_cmd.py

@@ -169,7 +169,7 @@ def chunkit(chunker_name, *args, **kwargs):
         ]:
             print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
 
-        from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
+        from ..crypto.low_level import AES256_CTR_BLAKE2b_legacy, AES256_CTR_HMAC_SHA256
         from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305
 
         print("Encryption =====================================================")
@@ -184,7 +184,7 @@ def chunkit(chunker_name, *args, **kwargs):
             ),
             (
                 "aes-256-ctr-blake2b",
-                lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
+                lambda: AES256_CTR_BLAKE2b_legacy(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
                     random_10M, header=b"X"
                 ),
             ),

src/borg/constants.py

@@ -175,7 +175,7 @@ class KeyType:
     REPO = 0x03
     BLAKE2KEYFILE = 0x04
     BLAKE2REPO = 0x05
-    BLAKE2AUTHENTICATED = 0x06
+    BLAKE2AUTHENTICATEDLEGACY = 0x06
     AUTHENTICATED = 0x07
     # new crypto
     # upper 4 bits are ciphersuite, lower 4 bits are keytype
@@ -187,6 +187,7 @@ class KeyType:
     BLAKE2AESOCBREPO = 0x31
     BLAKE2CHPOKEYFILE = 0x40
     BLAKE2CHPOREPO = 0x41
+    BLAKE2AUTHENTICATED = 0x51
 
 
 CACHE_TAG_NAME = "CACHEDIR.TAG"

src/borg/crypto/key.py

@@ -28,7 +28,7 @@
 
 
 from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
-from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305
+from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b_legacy, AES256_OCB, CHACHA20_POLY1305
 from . import low_level
 
 # workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode
@@ -122,7 +122,7 @@ def uses_same_id_hash(other_key, key):
     new_sha256_ids = (PlaintextKey,)
     old_hmac_sha256_ids = (RepoKey, KeyfileKey, AuthenticatedKey)
     new_hmac_sha256_ids = (AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey, AuthenticatedKey)
-    old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey, Blake2AuthenticatedKey)
+    old_blake2_ids = tuple()  # empty tuple, old blake2 IDs are incompatible with new blake2 IDs
     new_blake2_ids = (
         Blake2AESOCBRepoKey,
         Blake2AESOCBKeyfileKey,
@@ -251,7 +251,7 @@ def decrypt(self, id, data):
         return memoryview(data)[1:]
 
 
-def random_blake2b_256_key():
+def random_blake2b_256_key_legacy():  # borg 1.x created the key this way
     # This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b.
     # Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b
     # has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a
@@ -265,22 +265,36 @@ def random_blake2b_256_key():
     return os.urandom(64) + bytes(64)
 
 
-class ID_BLAKE2b_256:
+class ID_BLAKE2b_256_legacy:  # borg 1.x
     """
     Key mix-in class for using BLAKE2b-256 for the id key.
+    """
 
-    The id_key length must be 32 bytes.
+    def id_hash(self, data):
+        return blake2b_256(self.id_key, data, legacy=True)
+
+    def init_from_random_data(self):
+        super().init_from_random_data()
+        enc_key = os.urandom(32)
+        enc_hmac_key = random_blake2b_256_key_legacy()
+        self.crypt_key = enc_key + enc_hmac_key
+        self.id_key = random_blake2b_256_key_legacy()
+
+
+class ID_BLAKE2b_256:  # borg 2: either use the "fixed" blake2b or use blake3? see #8867
+    """
+    Key mix-in class for using BLAKE2b-256 for the id key.
     """
 
     def id_hash(self, data):
-        return blake2b_256(self.id_key, data)
+        return blake2b_256(self.id_key, data, legacy=False)
 
     def init_from_random_data(self):
         super().init_from_random_data()
         enc_key = os.urandom(32)
-        enc_hmac_key = random_blake2b_256_key()
+        enc_hmac_key = os.urandom(64)
         self.crypt_key = enc_key + enc_hmac_key
-        self.id_key = random_blake2b_256_key()
+        self.id_key = os.urandom(64)
 
 
 class ID_HMAC_SHA_256:
@@ -722,22 +736,22 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
     CIPHERSUITE = AES256_CTR_HMAC_SHA256
 
 
-class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
-    TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
-    TYPE = KeyType.BLAKE2KEYFILE
-    NAME = "key file BLAKE2b"
-    ARG_NAME = "keyfile-blake2"
+class Blake2KeyfileKeyLegacy(ID_BLAKE2b_256_legacy, AESKeyBase, FlexiKey):
+    TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}  # ???
+    TYPE = KeyType.BLAKE2KEYFILE  # ???
+    NAME = "key file BLAKE2b (legacy)"
+    ARG_NAME = "keyfile-blake2-legacy"
     STORAGE = KeyBlobStorage.KEYFILE
-    CIPHERSUITE = AES256_CTR_BLAKE2b
+    CIPHERSUITE = AES256_CTR_BLAKE2b_legacy
 
 
-class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
-    TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
-    TYPE = KeyType.BLAKE2REPO
-    NAME = "repokey BLAKE2b"
-    ARG_NAME = "repokey-blake2"
+class Blake2RepoKeyLegacy(ID_BLAKE2b_256_legacy, AESKeyBase, FlexiKey):
+    TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}  # ???
+    TYPE = KeyType.BLAKE2REPO  # ???
+    NAME = "repokey BLAKE2b (legacy)"
+    ARG_NAME = "repokey-blake2-legacy"
     STORAGE = KeyBlobStorage.REPO
-    CIPHERSUITE = AES256_CTR_BLAKE2b
+    CIPHERSUITE = AES256_CTR_BLAKE2b_legacy
 
 
 class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
@@ -786,16 +800,23 @@ class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
     ARG_NAME = "authenticated"
 
 
+class Blake2AuthenticatedKeyLegacy(ID_BLAKE2b_256_legacy, AuthenticatedKeyBase):
+    TYPE = KeyType.BLAKE2AUTHENTICATEDLEGACY
+    TYPES_ACCEPTABLE = {TYPE}
+    NAME = "authenticated BLAKE2b (legacy)"
+    ARG_NAME = "authenticated-blake2-legacy"
+
+
+# ------------ new crypto ------------
+
+
 class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase):
     TYPE = KeyType.BLAKE2AUTHENTICATED
     TYPES_ACCEPTABLE = {TYPE}
     NAME = "authenticated BLAKE2b"
     ARG_NAME = "authenticated-blake2"
 
 
-# ------------ new crypto ------------
-
-
 class AEADKeyBase(KeyBase):
     """
     Chunks are encrypted and authenticated using some AEAD ciphersuite
@@ -980,24 +1001,25 @@ class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
 
 
 LEGACY_KEY_TYPES = (
-    # legacy (AES-CTR based) crypto
+    # legacy (AES-CTR or Blake2_legacy based) crypto
     KeyfileKey,
     RepoKey,
-    Blake2KeyfileKey,
-    Blake2RepoKey,
+    Blake2KeyfileKeyLegacy,
+    Blake2RepoKeyLegacy,
+    Blake2AuthenticatedKeyLegacy,
 )
 
 AVAILABLE_KEY_TYPES = (
     # these are available encryption modes for new repositories
     # not encrypted modes
     PlaintextKey,
     AuthenticatedKey,
-    Blake2AuthenticatedKey,
     # new crypto
     AESOCBKeyfileKey,
     AESOCBRepoKey,
     CHPOKeyfileKey,
     CHPORepoKey,
+    Blake2AuthenticatedKey,
     Blake2AESOCBKeyfileKey,
     Blake2AESOCBRepoKey,
     Blake2CHPOKeyfileKey,

src/borg/crypto/low_level.pyx

@@ -364,7 +364,7 @@ cdef class AES256_CTR_HMAC_SHA256(AES256_CTR_BASE):
             raise IntegrityError('MAC Authentication failed')
 
 
-cdef class AES256_CTR_BLAKE2b(AES256_CTR_BASE):
+cdef class AES256_CTR_BLAKE2b_legacy(AES256_CTR_BASE):
     cdef unsigned char mac_key[128]
 
     def __init__(self, mac_key, enc_key, iv=None, header_len=1, aad_offset=1):
@@ -708,8 +708,13 @@ def hmac_sha256(key, data):
     return hmac.digest(key, data, 'sha256')
 
 
-def blake2b_256(key, data):
-    return hashlib.blake2b(key+data, digest_size=32).digest()
+def blake2b_256(key, data, legacy=False):
+    if legacy:
+        assert len(key) in (0, 128)  # borg 1.x 64B key + 64B zero padding (b"" used by tests)
+        return hashlib.blake2b(key+data, digest_size=32).digest()
+    else:
+        assert len(key) == 64  # borg 2.x 64B key
+        return hashlib.blake2b(data, key=key, digest_size=32).digest()
 
 
 def blake2b_128(data):

src/borg/testsuite/chunker_slow_test.py

@@ -28,10 +28,10 @@ def twist(size):
                     for seed in (1849058162, 1234567653):
                         fh = BytesIO(data)
                         chunker = Chunker(seed, minexp, maxexp, maskbits, winsize)
-                        chunks = [blake2b_256(b"", c) for c in cf(chunker.chunkify(fh, -1))]
-                        runs.append(blake2b_256(b"", b"".join(chunks)))
+                        chunks = [blake2b_256(b"", c, legacy=True) for c in cf(chunker.chunkify(fh, -1))]
+                        runs.append(blake2b_256(b"", b"".join(chunks), legacy=True))
 
     # The "correct" hash below matches the existing chunker behavior.
     # Future chunker optimisations must not change this, or existing repos will bloat.
-    overall_hash = blake2b_256(b"", b"".join(runs))
+    overall_hash = blake2b_256(b"", b"".join(runs), legacy=True)
     assert overall_hash == hex_to_bin("b559b0ac8df8daaa221201d018815114241ea5c6609d98913cd2246a702af4e3")

src/borg/testsuite/key_test.py

@@ -4,12 +4,13 @@
 
 import pytest
 
-from ..crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKey
-from ..crypto.key import RepoKey, KeyfileKey, Blake2RepoKey, Blake2KeyfileKey
+from ..crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKeyLegacy
+from ..crypto.key import RepoKey, KeyfileKey, Blake2RepoKeyLegacy, Blake2KeyfileKeyLegacy
 from ..crypto.key import AEADKeyBase
 from ..crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey
 from ..crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey
-from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256
+from ..crypto.key import Blake2AuthenticatedKey
+from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE2b_256_legacy
 from ..crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError
 from ..crypto.key import identify_key
 from ..crypto.low_level import IntegrityError as IntegrityErrorBase
@@ -40,7 +41,7 @@ class MockArgs:
     )
     keyfile2_id = hex_to_bin("c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314")
 
-    keyfile_blake2_key_file = """
+    keyfile_blake2_key_file_legacy = """
         BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000
         hqlhbGdvcml0aG2mc2hhMjU2pGRhdGHaAZ7VCsTjbLhC1ipXOyhcGn7YnROEhP24UQvOCi
         Oar1G+JpwgO9BIYaiCODUpzPuDQEm6WxyTwEneJ3wsuyeqyh7ru2xo9FAUKRf6jcqqZnan
@@ -54,17 +55,17 @@ class MockArgs:
         UTHFJg343jqml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgz3YaUZZ/s+UWywj97EY5b4KhtJYi
         qkPqtDDxs2j/T7+ndmVyc2lvbgE=""".strip()
 
-    keyfile_blake2_cdata = hex_to_bin(
+    keyfile_blake2_cdata_legacy = hex_to_bin(
         "04d6040f5ef80e0a8ac92badcbe3dee83b7a6b53d5c9a58c4eed14964cb10ef591040404040404040d1e65cc1f435027"
     )
     # Verified against b2sum. Entire string passed to BLAKE2, including the padded 64 byte key contained in
-    # keyfile_blake2_key_file above is
+    # keyfile_blake2_key_file_legacy above is
     # 19280471de95185ec27ecb6fc9edbb4f4db26974c315ede1cd505fab4250ce7cd0d081ea66946c
     # 95f0db934d5f616921efbd869257e8ded2bd9bd93d7f07b1a30000000000000000000000000000
     # 000000000000000000000000000000000000000000000000000000000000000000000000000000
     # 00000000000000000000007061796c6f6164
     #                       p a y l o a d
-    keyfile_blake2_id = hex_to_bin("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb")
+    keyfile_blake2_id_legacy = hex_to_bin("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb")
 
     @pytest.fixture
     def keys_dir(self, request, monkeypatch, tmpdir):
@@ -76,13 +77,14 @@ def keys_dir(self, request, monkeypatch, tmpdir):
             # not encrypted
             PlaintextKey,
             AuthenticatedKey,
-            Blake2AuthenticatedKey,
             # legacy crypto
+            Blake2AuthenticatedKeyLegacy,
             KeyfileKey,
-            Blake2KeyfileKey,
+            Blake2KeyfileKeyLegacy,
             RepoKey,
-            Blake2RepoKey,
+            Blake2RepoKeyLegacy,
             # new crypto
+            Blake2AuthenticatedKey,
             AESOCBKeyfileKey,
             AESOCBRepoKey,
             Blake2AESOCBKeyfileKey,
@@ -176,12 +178,12 @@ def test_keyfile2_kfenv(self, tmpdir, monkeypatch):
         key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
         assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload"
 
-    def test_keyfile_blake2(self, monkeypatch, keys_dir):
+    def test_keyfile_blake2_legacy(self, monkeypatch, keys_dir):
         with keys_dir.join("keyfile").open("w") as fd:
-            fd.write(self.keyfile_blake2_key_file)
+            fd.write(self.keyfile_blake2_key_file_legacy)
         monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
-        key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata)
-        assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b"payload"
+        key = Blake2KeyfileKeyLegacy.detect(self.MockRepository(), self.keyfile_blake2_cdata_legacy)
+        assert key.decrypt(self.keyfile_blake2_id_legacy, self.keyfile_blake2_cdata_legacy) == b"payload"
 
     def _corrupt_byte(self, key, data, offset):
         data = bytearray(data)
@@ -243,17 +245,28 @@ def test_authenticated_encrypt(self, monkeypatch):
         # 0x07 is the key TYPE.
         assert authenticated == b"\x07" + plaintext
 
-    def test_blake2_authenticated_encrypt(self, monkeypatch):
+    def test_blake2_authenticated_encrypt_legacy(self, monkeypatch):
         monkeypatch.setenv("BORG_PASSPHRASE", "test")
-        key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
-        assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash
+        key = Blake2AuthenticatedKeyLegacy.create(self.MockRepository(), self.MockArgs())
+        assert Blake2AuthenticatedKeyLegacy.id_hash is ID_BLAKE2b_256_legacy.id_hash
         assert len(key.id_key) == 128
         plaintext = b"123456789"
         id = key.id_hash(plaintext)
         authenticated = key.encrypt(id, plaintext)
         # 0x06 is the key TYPE.
         assert authenticated == b"\x06" + plaintext
 
+    def test_blake2_authenticated_encrypt(self, monkeypatch):
+        monkeypatch.setenv("BORG_PASSPHRASE", "test")
+        key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
+        assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash
+        assert len(key.id_key) == 64
+        plaintext = b"123456789"
+        id = key.id_hash(plaintext)
+        authenticated = key.encrypt(id, plaintext)
+        # 0x51 is the key TYPE.
+        assert authenticated == b"\x51" + plaintext
+
 
 class TestTAM:
     @pytest.fixture