From 8e85f90cc2bf5fe9a0b6cb8db6f4bcf9ac08ddca Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 29 May 2025 01:12:29 +0200 Subject: [PATCH] blake2b_256: fix key passing, fixes #8867 borg 1.x involved the key a bit unusually in the blake2b hash calculation (padding 64B of key material with another 64B of zeros to a total of 128B and then just prepending it to the data). since a while, we use blake2b from python stdlib and they support passing the key as a separate argument. up to 64B keys are allowed here. besides the way how the 64B of key material are passed, there are also other differences in how the key gets involved in the digest computation, thus the legacy blake2b hashes from borg 1.x are not compatible with the non-legacy ones - the hash is not the same for same key and data values. --- src/borg/archiver/benchmark_cmd.py | 4 +- src/borg/constants.py | 3 +- src/borg/crypto/key.py | 76 +++++++++++++++++---------- src/borg/crypto/low_level.pyx | 11 ++-- src/borg/testsuite/crypto/key_test.py | 47 +++++++++++------ 5 files changed, 91 insertions(+), 50 deletions(-) diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py index b1b241c4a3..32a460860c 100644 --- a/src/borg/archiver/benchmark_cmd.py +++ b/src/borg/archiver/benchmark_cmd.py @@ -180,7 +180,7 @@ def chunkit(ch): ]: print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") - from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256 + from ..crypto.low_level import AES256_CTR_BLAKE2b_legacy, AES256_CTR_HMAC_SHA256 from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305 print("Encryption =====================================================") @@ -195,7 +195,7 @@ def chunkit(ch): ), ( "aes-256-ctr-blake2b", - lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt( + lambda: AES256_CTR_BLAKE2b_legacy(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt( random_10M, header=b"X" ), ), diff --git a/src/borg/constants.py b/src/borg/constants.py index 092ca771f9..3e4c5438c0 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -179,7 +179,7 @@ class KeyType: REPO = 0x03 BLAKE2KEYFILE = 0x04 BLAKE2REPO = 0x05 - BLAKE2AUTHENTICATED = 0x06 + BLAKE2AUTHENTICATEDLEGACY = 0x06 AUTHENTICATED = 0x07 # new crypto # upper 4 bits are ciphersuite, lower 4 bits are keytype @@ -191,6 +191,7 @@ class KeyType: BLAKE2AESOCBREPO = 0x31 BLAKE2CHPOKEYFILE = 0x40 BLAKE2CHPOREPO = 0x41 + BLAKE2AUTHENTICATED = 0x51 CACHE_TAG_NAME = "CACHEDIR.TAG" diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 1b8e000447..72d0d270b3 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -29,7 +29,7 @@ from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256 -from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305 +from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b_legacy, AES256_OCB, CHACHA20_POLY1305 from . import low_level # workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode @@ -123,7 +123,7 @@ def uses_same_id_hash(other_key, key): new_sha256_ids = (PlaintextKey,) old_hmac_sha256_ids = (RepoKey, KeyfileKey, AuthenticatedKey) new_hmac_sha256_ids = (AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey, AuthenticatedKey) - old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey, Blake2AuthenticatedKey) + old_blake2_ids = tuple() # empty tuple, old blake2 IDs are incompatible with new blake2 IDs new_blake2_ids = ( Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, @@ -276,7 +276,7 @@ def decrypt(self, id, data): return memoryview(data)[1:] -def random_blake2b_256_key(): +def random_blake2b_256_key_legacy(): # borg 1.x created the key this way # This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b. # Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b # has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a @@ -290,22 +290,36 @@ def random_blake2b_256_key(): return os.urandom(64) + bytes(64) -class ID_BLAKE2b_256: +class ID_BLAKE2b_256_legacy: # borg 1.x """ Key mix-in class for using BLAKE2b-256 for the id key. + """ - The id_key length must be 32 bytes. + def id_hash(self, data): + return blake2b_256(self.id_key, data, legacy=True) + + def init_from_random_data(self): + super().init_from_random_data() + enc_key = os.urandom(32) + enc_hmac_key = random_blake2b_256_key_legacy() + self.crypt_key = enc_key + enc_hmac_key + self.id_key = random_blake2b_256_key_legacy() + + +class ID_BLAKE2b_256: # borg 2: either use the "fixed" blake2b or use blake3? see #8867 + """ + Key mix-in class for using BLAKE2b-256 for the id key. """ def id_hash(self, data): - return blake2b_256(self.id_key, data) + return blake2b_256(self.id_key, data, legacy=False) def init_from_random_data(self): super().init_from_random_data() enc_key = os.urandom(32) - enc_hmac_key = random_blake2b_256_key() + enc_hmac_key = os.urandom(64) self.crypt_key = enc_key + enc_hmac_key - self.id_key = random_blake2b_256_key() + self.id_key = os.urandom(64) class ID_HMAC_SHA_256: @@ -747,22 +761,22 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): CIPHERSUITE = AES256_CTR_HMAC_SHA256 -class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} - TYPE = KeyType.BLAKE2KEYFILE - NAME = "key file BLAKE2b" - ARG_NAME = "keyfile-blake2" +class Blake2KeyfileKeyLegacy(ID_BLAKE2b_256_legacy, AESKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} # ??? + TYPE = KeyType.BLAKE2KEYFILE # ??? + NAME = "key file BLAKE2b (legacy)" + ARG_NAME = "keyfile-blake2-legacy" STORAGE = KeyBlobStorage.KEYFILE - CIPHERSUITE = AES256_CTR_BLAKE2b + CIPHERSUITE = AES256_CTR_BLAKE2b_legacy -class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} - TYPE = KeyType.BLAKE2REPO - NAME = "repokey BLAKE2b" - ARG_NAME = "repokey-blake2" +class Blake2RepoKeyLegacy(ID_BLAKE2b_256_legacy, AESKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} # ??? + TYPE = KeyType.BLAKE2REPO # ??? + NAME = "repokey BLAKE2b (legacy)" + ARG_NAME = "repokey-blake2-legacy" STORAGE = KeyBlobStorage.REPO - CIPHERSUITE = AES256_CTR_BLAKE2b + CIPHERSUITE = AES256_CTR_BLAKE2b_legacy class AuthenticatedKeyBase(AESKeyBase, FlexiKey): @@ -811,6 +825,16 @@ class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase): ARG_NAME = "authenticated" +class Blake2AuthenticatedKeyLegacy(ID_BLAKE2b_256_legacy, AuthenticatedKeyBase): + TYPE = KeyType.BLAKE2AUTHENTICATEDLEGACY + TYPES_ACCEPTABLE = {TYPE} + NAME = "authenticated BLAKE2b (legacy)" + ARG_NAME = "authenticated-blake2-legacy" + + +# ------------ new crypto ------------ + + class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): TYPE = KeyType.BLAKE2AUTHENTICATED TYPES_ACCEPTABLE = {TYPE} @@ -818,9 +842,6 @@ class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): ARG_NAME = "authenticated-blake2" -# ------------ new crypto ------------ - - class AEADKeyBase(KeyBase): """ Chunks are encrypted and authenticated using some AEAD ciphersuite @@ -1003,11 +1024,12 @@ class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): LEGACY_KEY_TYPES = ( - # legacy (AES-CTR based) crypto + # legacy (AES-CTR or Blake2_legacy based) crypto KeyfileKey, RepoKey, - Blake2KeyfileKey, - Blake2RepoKey, + Blake2KeyfileKeyLegacy, + Blake2RepoKeyLegacy, + Blake2AuthenticatedKeyLegacy, ) AVAILABLE_KEY_TYPES = ( @@ -1015,12 +1037,12 @@ class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): # not encrypted modes PlaintextKey, AuthenticatedKey, - Blake2AuthenticatedKey, # new crypto AESOCBKeyfileKey, AESOCBRepoKey, CHPOKeyfileKey, CHPORepoKey, + Blake2AuthenticatedKey, Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey, Blake2CHPOKeyfileKey, diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index f723cde52b..50dbf87c16 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -368,7 +368,7 @@ cdef class AES256_CTR_HMAC_SHA256(AES256_CTR_BASE): raise IntegrityError('MAC Authentication failed') -cdef class AES256_CTR_BLAKE2b(AES256_CTR_BASE): +cdef class AES256_CTR_BLAKE2b_legacy(AES256_CTR_BASE): cdef unsigned char mac_key[128] def __init__(self, mac_key, enc_key, iv=None, header_len=1, aad_offset=1): @@ -712,8 +712,13 @@ def hmac_sha256(key, data): return hmac.digest(key, data, 'sha256') -def blake2b_256(key, data): - return hashlib.blake2b(key+data, digest_size=32).digest() +def blake2b_256(key, data, legacy=False): + if legacy: + assert len(key) in (0, 128) # borg 1.x 64B key + 64B zero padding (b"" used by tests) + return hashlib.blake2b(key+data, digest_size=32).digest() + else: + assert len(key) == 64 # borg 2.x 64B key + return hashlib.blake2b(data, key=key, digest_size=32).digest() def blake2b_128(data): diff --git a/src/borg/testsuite/crypto/key_test.py b/src/borg/testsuite/crypto/key_test.py index ca2884e299..32884f36f7 100644 --- a/src/borg/testsuite/crypto/key_test.py +++ b/src/borg/testsuite/crypto/key_test.py @@ -4,12 +4,13 @@ import pytest -from ...crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKey -from ...crypto.key import RepoKey, KeyfileKey, Blake2RepoKey, Blake2KeyfileKey +from ...crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKeyLegacy +from ...crypto.key import RepoKey, KeyfileKey, Blake2RepoKeyLegacy, Blake2KeyfileKeyLegacy from ...crypto.key import AEADKeyBase from ...crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey from ...crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey -from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256 +from ...crypto.key import Blake2AuthenticatedKey +from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE2b_256_legacy from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError from ...crypto.key import identify_key from ...crypto.low_level import IntegrityError as IntegrityErrorBase @@ -40,7 +41,7 @@ class MockArgs: ) keyfile2_id = hex_to_bin("c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314") - keyfile_blake2_key_file = """ + keyfile_blake2_key_file_legacy = """ BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000 hqlhbGdvcml0aG2mc2hhMjU2pGRhdGHaAZ7VCsTjbLhC1ipXOyhcGn7YnROEhP24UQvOCi Oar1G+JpwgO9BIYaiCODUpzPuDQEm6WxyTwEneJ3wsuyeqyh7ru2xo9FAUKRf6jcqqZnan @@ -54,17 +55,17 @@ class MockArgs: UTHFJg343jqml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgz3YaUZZ/s+UWywj97EY5b4KhtJYi qkPqtDDxs2j/T7+ndmVyc2lvbgE=""".strip() - keyfile_blake2_cdata = hex_to_bin( + keyfile_blake2_cdata_legacy = hex_to_bin( "04d6040f5ef80e0a8ac92badcbe3dee83b7a6b53d5c9a58c4eed14964cb10ef591040404040404040d1e65cc1f435027" ) # Verified against b2sum. Entire string passed to BLAKE2, including the padded 64 byte key contained in - # keyfile_blake2_key_file above is + # keyfile_blake2_key_file_legacy above is # 19280471de95185ec27ecb6fc9edbb4f4db26974c315ede1cd505fab4250ce7cd0d081ea66946c # 95f0db934d5f616921efbd869257e8ded2bd9bd93d7f07b1a30000000000000000000000000000 # 000000000000000000000000000000000000000000000000000000000000000000000000000000 # 00000000000000000000007061796c6f6164 # p a y l o a d - keyfile_blake2_id = hex_to_bin("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb") + keyfile_blake2_id_legacy = hex_to_bin("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb") @pytest.fixture def keys_dir(self, request, monkeypatch, tmpdir): @@ -76,13 +77,14 @@ def keys_dir(self, request, monkeypatch, tmpdir): # not encrypted PlaintextKey, AuthenticatedKey, - Blake2AuthenticatedKey, # legacy crypto + Blake2AuthenticatedKeyLegacy, KeyfileKey, - Blake2KeyfileKey, + Blake2KeyfileKeyLegacy, RepoKey, - Blake2RepoKey, + Blake2RepoKeyLegacy, # new crypto + Blake2AuthenticatedKey, AESOCBKeyfileKey, AESOCBRepoKey, Blake2AESOCBKeyfileKey, @@ -176,12 +178,12 @@ def test_keyfile2_kfenv(self, tmpdir, monkeypatch): key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload" - def test_keyfile_blake2(self, monkeypatch, keys_dir): + def test_keyfile_blake2_legacy(self, monkeypatch, keys_dir): with keys_dir.join("keyfile").open("w") as fd: - fd.write(self.keyfile_blake2_key_file) + fd.write(self.keyfile_blake2_key_file_legacy) monkeypatch.setenv("BORG_PASSPHRASE", "passphrase") - key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata) - assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b"payload" + key = Blake2KeyfileKeyLegacy.detect(self.MockRepository(), self.keyfile_blake2_cdata_legacy) + assert key.decrypt(self.keyfile_blake2_id_legacy, self.keyfile_blake2_cdata_legacy) == b"payload" def _corrupt_byte(self, key, data, offset): data = bytearray(data) @@ -243,10 +245,10 @@ def test_authenticated_encrypt(self, monkeypatch): # 0x07 is the key TYPE. assert authenticated == b"\x07" + plaintext - def test_blake2_authenticated_encrypt(self, monkeypatch): + def test_blake2_authenticated_encrypt_legacy(self, monkeypatch): monkeypatch.setenv("BORG_PASSPHRASE", "test") - key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) - assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash + key = Blake2AuthenticatedKeyLegacy.create(self.MockRepository(), self.MockArgs()) + assert Blake2AuthenticatedKeyLegacy.id_hash is ID_BLAKE2b_256_legacy.id_hash assert len(key.id_key) == 128 plaintext = b"123456789" id = key.id_hash(plaintext) @@ -254,6 +256,17 @@ def test_blake2_authenticated_encrypt(self, monkeypatch): # 0x06 is the key TYPE. assert authenticated == b"\x06" + plaintext + def test_blake2_authenticated_encrypt(self, monkeypatch): + monkeypatch.setenv("BORG_PASSPHRASE", "test") + key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) + assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash + assert len(key.id_key) == 64 + plaintext = b"123456789" + id = key.id_hash(plaintext) + authenticated = key.encrypt(id, plaintext) + # 0x51 is the key TYPE. + assert authenticated == b"\x51" + plaintext + class TestTAM: @pytest.fixture