diff --git a/encryption/path_test.go b/encryption/path_test.go index cdf2cb19e..2f3cc5fd0 100644 --- a/encryption/path_test.go +++ b/encryption/path_test.go @@ -20,7 +20,7 @@ import ( func newStore(key storj.Key, pathCipher storj.CipherSuite) *Store { store := NewStore() - if err := store.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, key, pathCipher); err != nil { + if err := store.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, key, pathCipher, storj.EncNull); err != nil { panic(err) } return store @@ -128,7 +128,7 @@ func TestStoreEncryption_BucketRoot(t *testing.T) { if !assert.NoError(t, err, errTag) { continue } - err = bucketStore.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, *bucketKey, cipher) + err = bucketStore.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, *bucketKey, cipher, storj.EncNull) if !assert.NoError(t, err, errTag) { continue } @@ -187,7 +187,7 @@ func TestStoreEncryption_MultipleBases(t *testing.T) { encPrefix, err := EncryptPath("bucket", prefix, cipher, rootStore) require.NoError(t, err) - err = prefixStore.AddWithCipher("bucket", prefix, encPrefix, *prefixKey, cipher) + err = prefixStore.AddWithCipher("bucket", prefix, encPrefix, *prefixKey, cipher, storj.EncNull) require.NoError(t, err) path := paths.NewUnencrypted(rawPath) diff --git a/encryption/store.go b/encryption/store.go index f661e26f5..2d5306e9c 100644 --- a/encryption/store.go +++ b/encryption/store.go @@ -32,9 +32,10 @@ import ( // b1, u6/u7 => <{e8:u8}, [u7], > // b2, u1 => <{}, [u1], > type Store struct { - roots map[string]*node - defaultKey *storj.Key - defaultPathCipher storj.CipherSuite + roots map[string]*node + defaultKey *storj.Key + defaultPathCipher storj.CipherSuite + defaultMetadataCipher storj.CipherSuite // EncryptionBypass makes it so we can interoperate with // the network without having encryption keys. paths will be encrypted but @@ -51,9 +52,10 @@ func (s *Store) Clone() *Store { } clone := &Store{ - roots: make(map[string]*node), - defaultPathCipher: s.defaultPathCipher, - EncryptionBypass: s.EncryptionBypass, + roots: make(map[string]*node), + defaultPathCipher: s.defaultPathCipher, + defaultMetadataCipher: s.defaultMetadataCipher, + EncryptionBypass: s.EncryptionBypass, } // Deep copy the defaultKey if it's not nil @@ -120,11 +122,12 @@ func (n *node) clone() *node { // Base represents a key with which to derive further keys at some encrypted/unencrypted path. type Base struct { - Unencrypted paths.Unencrypted - Encrypted paths.Encrypted - Key storj.Key - PathCipher storj.CipherSuite - Default bool + Unencrypted paths.Unencrypted + Encrypted paths.Encrypted + Key storj.Key + PathCipher storj.CipherSuite + MetadataCipher storj.CipherSuite + Default bool } // clone returns a copy of the Base. The implementation can be simple because the @@ -172,13 +175,23 @@ func (s *Store) GetDefaultPathCipher() storj.CipherSuite { return s.defaultPathCipher } +// SetDefaultMetadataCipher adds a default metadata cipher to be returned for any lookup that does not match a bucket. +func (s *Store) SetDefaultMetadataCipher(defaultMetadataCipher storj.CipherSuite) { + s.defaultMetadataCipher = defaultMetadataCipher +} + +// GetDefaultMetadataCipher returns the default metadata cipher, or EncUnspecified if none has been set. +func (s *Store) GetDefaultMetadataCipher() storj.CipherSuite { + return s.defaultMetadataCipher +} + // Add creates a mapping from the unencrypted path to the encrypted path and key. It uses the current default cipher. func (s *Store) Add(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key) error { - return s.AddWithCipher(bucket, unenc, enc, key, s.defaultPathCipher) + return s.AddWithCipher(bucket, unenc, enc, key, s.defaultPathCipher, s.defaultMetadataCipher) } // AddWithCipher creates a mapping from the unencrypted path to the encrypted path and key with the given cipher. -func (s *Store) AddWithCipher(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error { +func (s *Store) AddWithCipher(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error { root, ok := s.roots[bucket] if !ok { root = newNode() @@ -186,10 +199,11 @@ func (s *Store) AddWithCipher(bucket string, unenc paths.Unencrypted, enc paths. // Perform the addition starting at the root node. if err := root.add(unenc.Iterator(), enc.Iterator(), &Base{ - Unencrypted: unenc, - Encrypted: enc, - Key: key, - PathCipher: pathCipher, + Unencrypted: unenc, + Encrypted: enc, + Key: key, + PathCipher: pathCipher, + MetadataCipher: metadataCipher, }); err != nil { return err } @@ -289,9 +303,10 @@ func (s *Store) LookupEncrypted(bucket string, path paths.Encrypted) ( func (s *Store) defaultBase() *Base { return &Base{ - Key: *s.defaultKey, - PathCipher: s.defaultPathCipher, - Default: true, + Key: *s.defaultKey, + PathCipher: s.defaultPathCipher, + MetadataCipher: s.defaultMetadataCipher, + Default: true, } } @@ -358,7 +373,7 @@ func (n *node) iterate(fn func(string, paths.Unencrypted, paths.Encrypted, storj } // IterateWithCipher executes the callback with every value that has been Added to the Store. -func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite) error) error { +func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite, storj.CipherSuite) error) error { for bucket, root := range s.roots { if err := root.iterateWithCipher(fn, bucket); err != nil { return err @@ -368,9 +383,9 @@ func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encry } // iterateWithCipher calls the callback if the node has a base, and recurses to its children. -func (n *node) iterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite) error, bucket string) error { +func (n *node) iterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite, storj.CipherSuite) error, bucket string) error { if n.base != nil { - err := fn(bucket, n.base.Unencrypted, n.base.Encrypted, n.base.Key, n.base.PathCipher) + err := fn(bucket, n.base.Unencrypted, n.base.Encrypted, n.base.Key, n.base.PathCipher, n.base.MetadataCipher) if err != nil { return err } diff --git a/encryption/store_test.go b/encryption/store_test.go index 8e1765960..520c15fd6 100644 --- a/encryption/store_test.go +++ b/encryption/store_test.go @@ -43,19 +43,27 @@ func abortIfError(err error) { } } +func forAllCipherPairs(test func(c1 storj.CipherSuite, c2 storj.CipherSuite)) { + forAllCiphers(func(c1 storj.CipherSuite) { + forAllCiphers(func(c2 storj.CipherSuite) { + test(c1, c2) + }) + }) +} + func ExampleStore() { s := NewStore() ep := paths.NewEncrypted up := paths.NewUnencrypted // Add a fairly complicated tree to the store. - abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b1", up("u6"), ep("e6"), toKey("k6"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b2", up("u1"), ep("e1'"), toKey("k1"), storj.EncAESGCM)) - abortIfError(s.AddWithCipher("b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u6"), ep("e6"), toKey("k6"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b2", up("u1"), ep("e1'"), toKey("k1"), storj.EncAESGCM, storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), storj.EncAESGCM, storj.EncAESGCM)) // Look up some complicated queries by the unencrypted path. printLookup(s.LookupUnencrypted("b1", up("u1"))) @@ -107,7 +115,7 @@ func ExampleStore_SetDefaultKey() { ep := paths.NewEncrypted up := paths.NewUnencrypted - abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM)) + abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM, storj.EncAESGCM)) printLookup(s.LookupUnencrypted("b1", up("u1"))) printLookup(s.LookupUnencrypted("b1", up("u1/u2"))) @@ -145,15 +153,15 @@ func TestStoreErrors(t *testing.T) { up := paths.NewUnencrypted // Too many encrypted parts - require.Error(t, s.AddWithCipher("b1", up("u1"), ep("e1/e2/e3"), storj.Key{}, pathCipher)) + require.Error(t, s.AddWithCipher("b1", up("u1"), ep("e1/e2/e3"), storj.Key{}, pathCipher, storj.EncNull)) // Too many unencrypted parts - require.Error(t, s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1"), storj.Key{}, pathCipher)) + require.Error(t, s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull)) // Mismatches - require.NoError(t, s.AddWithCipher("b1", up("u1"), ep("e1"), storj.Key{}, pathCipher)) - require.Error(t, s.AddWithCipher("b1", up("u2"), ep("e1"), storj.Key{}, pathCipher)) - require.Error(t, s.AddWithCipher("b1", up("u1"), ep("f1"), storj.Key{}, pathCipher)) + require.NoError(t, s.AddWithCipher("b1", up("u1"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull)) + require.Error(t, s.AddWithCipher("b1", up("u2"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull)) + require.Error(t, s.AddWithCipher("b1", up("u1"), ep("f1"), storj.Key{}, pathCipher, storj.EncNull)) } } @@ -166,9 +174,9 @@ func TestStoreErrorState(t *testing.T) { revealed1, consumed1, base1 := s.LookupUnencrypted("b1", up("u1/u2")) // Attempt to do an addition that fails. - require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncNull)) - require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncAESGCM)) - require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncSecretBox)) + require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncNull, storj.EncNull)) + require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncAESGCM, storj.EncAESGCM)) + require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncSecretBox, storj.EncSecretBox)) // Ensure that we get the same results as before revealed2, consumed2, base2 := s.LookupUnencrypted("b1", up("u1/u2")) @@ -180,18 +188,15 @@ func TestStoreErrorState(t *testing.T) { func TestStoreIterate(t *testing.T) { type storeEntry struct { - bucket string - unenc paths.Unencrypted - enc paths.Encrypted - key storj.Key - pathCipher storj.CipherSuite + bucket string + unenc paths.Unencrypted + enc paths.Encrypted + key storj.Key + pathCipher storj.CipherSuite + metadataCipher storj.CipherSuite } - for _, pathCipher := range []storj.CipherSuite{ - storj.EncNull, - storj.EncAESGCM, - storj.EncSecretBox, - } { + forAllCipherPairs(func(pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) { for _, bypass := range []bool{false, true} { s := NewStore() s.EncryptionBypass = bypass @@ -200,27 +205,27 @@ func TestStoreIterate(t *testing.T) { up := paths.NewUnencrypted expected := map[storeEntry]struct{}{ - {"b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), pathCipher}: {}, - {"b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), pathCipher}: {}, - {"b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), pathCipher}: {}, - {"b1", up("u6"), ep("e6"), toKey("k6"), pathCipher}: {}, - {"b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), pathCipher}: {}, - {"b2", up("u1"), ep("e1'"), toKey("k1"), pathCipher}: {}, - {"b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), pathCipher}: {}, + {"b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), pathCipher, metadataCipher}: {}, + {"b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), pathCipher, metadataCipher}: {}, + {"b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), pathCipher, metadataCipher}: {}, + {"b1", up("u6"), ep("e6"), toKey("k6"), pathCipher, metadataCipher}: {}, + {"b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), pathCipher, metadataCipher}: {}, + {"b2", up("u1"), ep("e1'"), toKey("k1"), pathCipher, metadataCipher}: {}, + {"b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), pathCipher, metadataCipher}: {}, } for entry := range expected { - require.NoError(t, s.AddWithCipher(entry.bucket, entry.unenc, entry.enc, entry.key, entry.pathCipher)) + require.NoError(t, s.AddWithCipher(entry.bucket, entry.unenc, entry.enc, entry.key, entry.pathCipher, entry.metadataCipher)) } got := make(map[storeEntry]struct{}) - require.NoError(t, s.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error { - got[storeEntry{bucket, unenc, enc, key, pathCipher}] = struct{}{} + require.NoError(t, s.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error { + got[storeEntry{bucket, unenc, enc, key, pathCipher, metadataCipher}] = struct{}{} return nil })) require.Equal(t, expected, got) } - } + }) } func TestStoreEncryptionBypass(t *testing.T) { @@ -248,6 +253,7 @@ func TestStoreClone(t *testing.T) { store := NewStore() store.SetDefaultKey(&defaultKey) store.SetDefaultPathCipher(storj.EncAESGCM) + store.SetDefaultMetadataCipher(storj.EncAESGCM) err := store.Add("bucket1", paths.NewUnencrypted("path1"), paths.NewEncrypted("encPath1"), pathKey) require.NoError(t, err) @@ -256,6 +262,7 @@ func TestStoreClone(t *testing.T) { assert.Equal(t, store, clone) assert.Equal(t, store.defaultPathCipher, clone.defaultPathCipher) + assert.Equal(t, store.defaultMetadataCipher, clone.defaultMetadataCipher) assert.Equal(t, store.EncryptionBypass, clone.EncryptionBypass) assert.NotSame(t, store.defaultKey, clone.defaultKey) diff --git a/grant/access.go b/grant/access.go index b1cb24fba..4b3643d0d 100644 --- a/grant/access.go +++ b/grant/access.go @@ -139,6 +139,11 @@ func (s *EncryptionAccess) SetDefaultPathCipher(defaultPathCipher storj.CipherSu s.Store.SetDefaultPathCipher(defaultPathCipher) } +// SetDefaultMetadataCipher sets which cipher suite to use by default for metadata. +func (s *EncryptionAccess) SetDefaultMetadataCipher(defaultMetadataCipher storj.CipherSuite) { + s.Store.SetDefaultMetadataCipher(defaultMetadataCipher) +} + // LimitTo limits the data in the encryption access only to the paths that would be // allowed by the api key. Any errors that happen due to the consistency of the api // key cause no keys to be stored. @@ -231,7 +236,8 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store, // create the new store that we'll load into and carry some necessary defaults store := encryption.NewStore() - store.SetDefaultPathCipher(s.Store.GetDefaultPathCipher()) // keep default path cipher + store.SetDefaultPathCipher(s.Store.GetDefaultPathCipher()) // keep default path cipher + store.SetDefaultMetadataCipher(s.Store.GetDefaultMetadataCipher()) // keep default metadata cipher // add the prefixes to the store, skipping any that fail for any reason for _, prefix := range prefixes { @@ -254,7 +260,7 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store, continue // this should not happen given Decrypt succeeded, but whatever } - if err := store.AddWithCipher(bucket, unencPath, encPath, *key, base.PathCipher); err != nil { + if err := store.AddWithCipher(bucket, unencPath, encPath, *key, base.PathCipher, base.MetadataCipher); err != nil { continue } } @@ -264,13 +270,14 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store, func (s *EncryptionAccess) toProto() (*pb.EncryptionAccess, error) { var storeEntries []*pb.EncryptionAccess_StoreEntry - err := s.Store.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error { + err := s.Store.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error { storeEntries = append(storeEntries, &pb.EncryptionAccess_StoreEntry{ Bucket: []byte(bucket), UnencryptedPath: []byte(unenc.Raw()), EncryptedPath: []byte(enc.Raw()), Key: key[:], PathCipher: pb.CipherSuite(pathCipher), + MetadataCipher: pb.CipherSuite(metadataCipher), }) return nil }) @@ -284,9 +291,10 @@ func (s *EncryptionAccess) toProto() (*pb.EncryptionAccess, error) { } return &pb.EncryptionAccess{ - DefaultKey: defaultKey, - StoreEntries: storeEntries, - DefaultPathCipher: pb.CipherSuite(s.Store.GetDefaultPathCipher()), + DefaultKey: defaultKey, + StoreEntries: storeEntries, + DefaultPathCipher: pb.CipherSuite(s.Store.GetDefaultPathCipher()), + DefaultMetadataCipher: pb.CipherSuite(s.Store.GetDefaultMetadataCipher()), }, nil } @@ -308,6 +316,11 @@ func parseEncryptionAccessFromProto(p *pb.EncryptionAccess) (*EncryptionAccess, access.SetDefaultPathCipher(storj.EncAESGCM) } + access.SetDefaultMetadataCipher(storj.CipherSuite(p.DefaultMetadataCipher)) + if p.DefaultMetadataCipher == pb.CipherSuite_ENC_UNSPECIFIED { + access.SetDefaultMetadataCipher(storj.EncAESGCM) + } + for _, entry := range p.StoreEntries { if len(entry.Key) != len(storj.Key{}) { return nil, errors.New("invalid key in encryption access entry") @@ -321,6 +334,7 @@ func parseEncryptionAccessFromProto(p *pb.EncryptionAccess) (*EncryptionAccess, paths.NewEncrypted(string(entry.EncryptedPath)), key, storj.CipherSuite(entry.PathCipher), + storj.CipherSuite(entry.MetadataCipher), ) if err != nil { return nil, fmt.Errorf("invalid encryption access entry: %w", err) diff --git a/grant/internal/pb/encryption_access.pico.go b/grant/internal/pb/encryption_access.pico.go index d1f0c882a..a2b73f23a 100644 --- a/grant/internal/pb/encryption_access.pico.go +++ b/grant/internal/pb/encryption_access.pico.go @@ -15,6 +15,7 @@ type EncryptionAccess struct { DefaultKey []byte `json:"default_key,omitempty"` StoreEntries []*EncryptionAccess_StoreEntry `json:"store_entries,omitempty"` DefaultPathCipher CipherSuite `json:"default_path_cipher,omitempty"` + DefaultMetadataCipher CipherSuite `json:"default_metadata_cipher,omitempty"` DefaultEncryptionParameters *EncryptionParameters `json:"default_encryption_parameters,omitempty"` } @@ -28,6 +29,7 @@ func (m *EncryptionAccess) Encode(c *picobuf.Encoder) bool { } c.Int32(3, (*int32)(&m.DefaultPathCipher)) c.Message(4, m.DefaultEncryptionParameters.Encode) + c.Int32(5, (*int32)(&m.DefaultMetadataCipher)) return true } @@ -48,6 +50,7 @@ func (m *EncryptionAccess) Decode(c *picobuf.Decoder) { } m.DefaultEncryptionParameters.Decode(c) }) + c.Int32(5, (*int32)(&m.DefaultMetadataCipher)) } type EncryptionAccess_StoreEntry struct { @@ -56,6 +59,7 @@ type EncryptionAccess_StoreEntry struct { EncryptedPath []byte `json:"encrypted_path,omitempty"` Key []byte `json:"key,omitempty"` PathCipher CipherSuite `json:"path_cipher,omitempty"` + MetadataCipher CipherSuite `json:"metadata_cipher,omitempty"` EncryptionParameters *EncryptionParameters `json:"encryption_parameters,omitempty"` } @@ -69,6 +73,7 @@ func (m *EncryptionAccess_StoreEntry) Encode(c *picobuf.Encoder) bool { c.Bytes(4, &m.Key) c.Int32(5, (*int32)(&m.PathCipher)) c.Message(6, m.EncryptionParameters.Encode) + c.Int32(7, (*int32)(&m.MetadataCipher)) return true } @@ -87,4 +92,5 @@ func (m *EncryptionAccess_StoreEntry) Decode(c *picobuf.Decoder) { } m.EncryptionParameters.Decode(c) }) + c.Int32(7, (*int32)(&m.MetadataCipher)) } diff --git a/pb/encryption_access.go b/pb/encryption_access.go index 807209478..ae28cd2d8 100644 --- a/pb/encryption_access.go +++ b/pb/encryption_access.go @@ -17,6 +17,7 @@ type encryptionAccessStoreEntryMarshal struct { EncryptedPath string `json:"encrypted_path,omitempty"` Key string `json:"key,omitempty"` PathCipher CipherSuite `json:"path_cipher,omitempty"` + MetadataCipher CipherSuite `json:"metadata_cipher,omitempty"` EncryptionParameters *EncryptionParameters `json:"encryption_parameters,omitempty"` } @@ -38,6 +39,7 @@ func (se *EncryptionAccess_StoreEntry) MarshalJSON() ([]byte, error) { EncryptedPath: path, Key: base64.URLEncoding.EncodeToString(se.Key), PathCipher: se.PathCipher, + MetadataCipher: se.MetadataCipher, EncryptionParameters: se.EncryptionParameters, }) } diff --git a/pb/encryption_access.pb.go b/pb/encryption_access.pb.go index eb0529aec..b199f0c8c 100644 --- a/pb/encryption_access.pb.go +++ b/pb/encryption_access.pb.go @@ -17,6 +17,7 @@ type EncryptionAccess struct { DefaultKey []byte `protobuf:"bytes,1,opt,name=default_key,json=defaultKey,proto3" json:"default_key,omitempty"` StoreEntries []*EncryptionAccess_StoreEntry `protobuf:"bytes,2,rep,name=store_entries,json=storeEntries,proto3" json:"store_entries,omitempty"` DefaultPathCipher CipherSuite `protobuf:"varint,3,opt,name=default_path_cipher,json=defaultPathCipher,proto3,enum=encryption.CipherSuite" json:"default_path_cipher,omitempty"` + DefaultMetadataCipher CipherSuite `protobuf:"varint,5,opt,name=default_metadata_cipher,json=defaultMetadataCipher,proto3,enum=encryption.CipherSuite" json:"default_metadata_cipher,omitempty"` DefaultEncryptionParameters *EncryptionParameters `protobuf:"bytes,4,opt,name=default_encryption_parameters,json=defaultEncryptionParameters,proto3" json:"default_encryption_parameters,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -66,6 +67,13 @@ func (m *EncryptionAccess) GetDefaultPathCipher() CipherSuite { return CipherSuite_ENC_UNSPECIFIED } +func (m *EncryptionAccess) GetDefaultMetadataCipher() CipherSuite { + if m != nil { + return m.DefaultMetadataCipher + } + return CipherSuite_ENC_UNSPECIFIED +} + func (m *EncryptionAccess) GetDefaultEncryptionParameters() *EncryptionParameters { if m != nil { return m.DefaultEncryptionParameters @@ -79,6 +87,7 @@ type EncryptionAccess_StoreEntry struct { EncryptedPath []byte `protobuf:"bytes,3,opt,name=encrypted_path,json=encryptedPath,proto3" json:"encrypted_path,omitempty"` Key []byte `protobuf:"bytes,4,opt,name=key,proto3" json:"key,omitempty"` PathCipher CipherSuite `protobuf:"varint,5,opt,name=path_cipher,json=pathCipher,proto3,enum=encryption.CipherSuite" json:"path_cipher,omitempty"` + MetadataCipher CipherSuite `protobuf:"varint,7,opt,name=metadata_cipher,json=metadataCipher,proto3,enum=encryption.CipherSuite" json:"metadata_cipher,omitempty"` EncryptionParameters *EncryptionParameters `protobuf:"bytes,6,opt,name=encryption_parameters,json=encryptionParameters,proto3" json:"encryption_parameters,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -142,6 +151,13 @@ func (m *EncryptionAccess_StoreEntry) GetPathCipher() CipherSuite { return CipherSuite_ENC_UNSPECIFIED } +func (m *EncryptionAccess_StoreEntry) GetMetadataCipher() CipherSuite { + if m != nil { + return m.MetadataCipher + } + return CipherSuite_ENC_UNSPECIFIED +} + func (m *EncryptionAccess_StoreEntry) GetEncryptionParameters() *EncryptionParameters { if m != nil { return m.EncryptionParameters diff --git a/pb/encryption_access.proto b/pb/encryption_access.proto index 031d861d7..0a16b8dc3 100644 --- a/pb/encryption_access.proto +++ b/pb/encryption_access.proto @@ -17,11 +17,13 @@ message EncryptionAccess { bytes key = 4; encryption.CipherSuite path_cipher = 5; + encryption.CipherSuite metadata_cipher = 7; encryption.EncryptionParameters encryption_parameters = 6; } bytes default_key = 1; repeated StoreEntry store_entries = 2; encryption.CipherSuite default_path_cipher = 3; + encryption.CipherSuite default_metadata_cipher = 5; encryption.EncryptionParameters default_encryption_parameters = 4; } diff --git a/pb/json_test.go b/pb/json_test.go index 658792af9..924039dc9 100644 --- a/pb/json_test.go +++ b/pb/json_test.go @@ -25,8 +25,9 @@ func TestMarshalJSON(t *testing.T) { EncryptedPath: []byte("enc"), Key: []byte("key"), PathCipher: CipherSuite_ENC_AESGCM, + MetadataCipher: CipherSuite_ENC_AESGCM, }) require.NoError(t, err) - require.Equal(t, string(bs), `{"bucket":"bucket","unencrypted_path":"unenc","encrypted_path":"ZW5j","key":"a2V5","path_cipher":"ENC_AESGCM"}`) + require.Equal(t, string(bs), `{"bucket":"bucket","unencrypted_path":"unenc","encrypted_path":"ZW5j","key":"a2V5","path_cipher":"ENC_AESGCM","metadata_cipher":"ENC_AESGCM"}`) }) } diff --git a/pb/metainfo.pb.go b/pb/metainfo.pb.go index a08a2a6b2..4c6b25a41 100644 --- a/pb/metainfo.pb.go +++ b/pb/metainfo.pb.go @@ -1522,6 +1522,7 @@ type Object struct { EncryptedMetadataNonce Nonce `protobuf:"bytes,9,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` EncryptedMetadata []byte `protobuf:"bytes,10,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` EncryptedMetadataEncryptedKey []byte `protobuf:"bytes,17,opt,name=encrypted_metadata_encrypted_key,json=encryptedMetadataEncryptedKey,proto3" json:"encrypted_metadata_encrypted_key,omitempty"` + ClearMetadata []byte `protobuf:"bytes,22,opt,name=clear_metadata,json=clearMetadata,proto3" json:"clear_metadata,omitempty"` // fixed_segment_size is 0 for migrated objects. FixedSegmentSize int64 `protobuf:"varint,11,opt,name=fixed_segment_size,json=fixedSegmentSize,proto3" json:"fixed_segment_size,omitempty"` RedundancyScheme *RedundancyScheme `protobuf:"bytes,12,opt,name=redundancy_scheme,json=redundancyScheme,proto3" json:"redundancy_scheme,omitempty"` @@ -1635,6 +1636,13 @@ func (m *Object) GetEncryptedMetadataEncryptedKey() []byte { return nil } +func (m *Object) GetClearMetadata() []byte { + if m != nil { + return m.ClearMetadata + } + return nil +} + func (m *Object) GetFixedSegmentSize() int64 { if m != nil { return m.FixedSegmentSize @@ -1709,6 +1717,7 @@ type BeginObjectRequest struct { EncryptedMetadataNonce Nonce `protobuf:"bytes,9,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` EncryptedMetadata []byte `protobuf:"bytes,10,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` EncryptedMetadataEncryptedKey []byte `protobuf:"bytes,11,opt,name=encrypted_metadata_encrypted_key,json=encryptedMetadataEncryptedKey,proto3" json:"encrypted_metadata_encrypted_key,omitempty"` + ClearMetadata []byte `protobuf:"bytes,16,opt,name=clear_metadata,json=clearMetadata,proto3" json:"clear_metadata,omitempty"` Retention *Retention `protobuf:"bytes,12,opt,name=retention,proto3" json:"retention,omitempty"` LegalHold bool `protobuf:"varint,13,opt,name=legal_hold,json=legalHold,proto3" json:"legal_hold,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1801,6 +1810,13 @@ func (m *BeginObjectRequest) GetEncryptedMetadataEncryptedKey() []byte { return nil } +func (m *BeginObjectRequest) GetClearMetadata() []byte { + if m != nil { + return m.ClearMetadata + } + return nil +} + func (m *BeginObjectRequest) GetRetention() *Retention { if m != nil { return m.Retention @@ -1895,6 +1911,7 @@ type CommitObjectRequest struct { EncryptedMetadataNonce Nonce `protobuf:"bytes,2,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` EncryptedMetadata []byte `protobuf:"bytes,3,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` EncryptedMetadataEncryptedKey []byte `protobuf:"bytes,4,opt,name=encrypted_metadata_encrypted_key,json=encryptedMetadataEncryptedKey,proto3" json:"encrypted_metadata_encrypted_key,omitempty"` + ClearMetadata []byte `protobuf:"bytes,16,opt,name=clear_metadata,json=clearMetadata,proto3" json:"clear_metadata,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1950,6 +1967,13 @@ func (m *CommitObjectRequest) GetEncryptedMetadataEncryptedKey() []byte { return nil } +func (m *CommitObjectRequest) GetClearMetadata() []byte { + if m != nil { + return m.ClearMetadata + } + return nil +} + type CommitObjectResponse struct { Object *Object `protobuf:"bytes,1,opt,name=object,proto3" json:"object,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2772,6 +2796,7 @@ type ObjectListItem struct { EncryptedMetadataNonce Nonce `protobuf:"bytes,7,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` EncryptedMetadataEncryptedKey []byte `protobuf:"bytes,11,opt,name=encrypted_metadata_encrypted_key,json=encryptedMetadataEncryptedKey,proto3" json:"encrypted_metadata_encrypted_key,omitempty"` EncryptedMetadata []byte `protobuf:"bytes,8,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` + ClearMetadata []byte `protobuf:"bytes,14,opt,name=clear_metadata,json=clearMetadata,proto3" json:"clear_metadata,omitempty"` // plain_size is 0 for migrated objects. PlainSize int64 `protobuf:"varint,10,opt,name=plain_size,json=plainSize,proto3" json:"plain_size,omitempty"` StreamId *StreamID `protobuf:"bytes,9,opt,name=stream_id,json=streamId,proto3,customtype=StreamID" json:"stream_id,omitempty"` @@ -2872,6 +2897,13 @@ func (m *ObjectListItem) GetEncryptedMetadata() []byte { return nil } +func (m *ObjectListItem) GetClearMetadata() []byte { + if m != nil { + return m.ClearMetadata + } + return nil +} + func (m *ObjectListItem) GetPlainSize() int64 { if m != nil { return m.PlainSize @@ -3259,6 +3291,7 @@ type UpdateObjectMetadataRequest struct { EncryptedMetadataNonce Nonce `protobuf:"bytes,4,opt,name=encrypted_metadata_nonce,json=encryptedMetadataNonce,proto3,customtype=Nonce" json:"encrypted_metadata_nonce"` EncryptedMetadata []byte `protobuf:"bytes,5,opt,name=encrypted_metadata,json=encryptedMetadata,proto3" json:"encrypted_metadata,omitempty"` EncryptedMetadataEncryptedKey []byte `protobuf:"bytes,6,opt,name=encrypted_metadata_encrypted_key,json=encryptedMetadataEncryptedKey,proto3" json:"encrypted_metadata_encrypted_key,omitempty"` + ClearMetadata []byte `protobuf:"bytes,16,opt,name=clear_metadata,json=clearMetadata,proto3" json:"clear_metadata,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3335,6 +3368,13 @@ func (m *UpdateObjectMetadataRequest) GetEncryptedMetadataEncryptedKey() []byte return nil } +func (m *UpdateObjectMetadataRequest) GetClearMetadata() []byte { + if m != nil { + return m.ClearMetadata + } + return nil +} + type UpdateObjectMetadataResponse struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -6694,6 +6734,7 @@ type FinishCopyObjectRequest struct { NewEncryptedMetadata []byte `protobuf:"bytes,7,opt,name=new_encrypted_metadata,json=newEncryptedMetadata,proto3" json:"new_encrypted_metadata,omitempty"` NewEncryptedMetadataKeyNonce Nonce `protobuf:"bytes,4,opt,name=new_encrypted_metadata_key_nonce,json=newEncryptedMetadataKeyNonce,proto3,customtype=Nonce" json:"new_encrypted_metadata_key_nonce"` NewEncryptedMetadataKey []byte `protobuf:"bytes,5,opt,name=new_encrypted_metadata_key,json=newEncryptedMetadataKey,proto3" json:"new_encrypted_metadata_key,omitempty"` + NewClearMetadata []byte `protobuf:"bytes,16,opt,name=new_clear_metadata,json=newClearMetadata,proto3" json:"new_clear_metadata,omitempty"` NewSegmentKeys []*EncryptedKeyAndNonce `protobuf:"bytes,6,rep,name=new_segment_keys,json=newSegmentKeys,proto3" json:"new_segment_keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -6785,6 +6826,13 @@ func (m *FinishCopyObjectRequest) GetNewEncryptedMetadataKey() []byte { return nil } +func (m *FinishCopyObjectRequest) GetNewClearMetadata() []byte { + if m != nil { + return m.NewClearMetadata + } + return nil +} + func (m *FinishCopyObjectRequest) GetNewSegmentKeys() []*EncryptedKeyAndNonce { if m != nil { return m.NewSegmentKeys diff --git a/pb/metainfo.proto b/pb/metainfo.proto index ebbf76f2a..bddf78f81 100644 --- a/pb/metainfo.proto +++ b/pb/metainfo.proto @@ -301,6 +301,7 @@ message Object { bytes encrypted_metadata_nonce = 9 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes encrypted_metadata = 10; bytes encrypted_metadata_encrypted_key = 17; + bytes clear_metadata = 22; // fixed_segment_size is 0 for migrated objects. int64 fixed_segment_size = 11; @@ -337,6 +338,7 @@ message BeginObjectRequest { bytes encrypted_metadata_nonce = 9 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes encrypted_metadata = 10; bytes encrypted_metadata_encrypted_key = 11; + bytes clear_metadata = 16; Retention retention = 12; bool legal_hold = 13; @@ -366,6 +368,7 @@ message CommitObjectRequest { bytes encrypted_metadata_nonce = 2 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes encrypted_metadata = 3; // TODO: set maximum size limit bytes encrypted_metadata_encrypted_key = 4; + bytes clear_metadata = 16; } message CommitObjectResponse { @@ -526,6 +529,7 @@ message ObjectListItem { bytes encrypted_metadata_nonce = 7 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes encrypted_metadata_encrypted_key = 11; bytes encrypted_metadata = 8; + bytes clear_metadata = 14; // plain_size is 0 for migrated objects. int64 plain_size = 10; @@ -597,6 +601,7 @@ message UpdateObjectMetadataRequest { bytes encrypted_metadata_nonce = 4 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes encrypted_metadata = 5; // TODO: set maximum size limit bytes encrypted_metadata_encrypted_key = 6; + bytes clear_metadata = 16; } message UpdateObjectMetadataResponse { @@ -1070,6 +1075,7 @@ message FinishCopyObjectRequest { bytes new_encrypted_metadata = 7; bytes new_encrypted_metadata_key_nonce = 4 [(gogoproto.customtype) = "Nonce", (gogoproto.nullable) = false]; bytes new_encrypted_metadata_key = 5; + bytes new_clear_metadata = 16; repeated EncryptedKeyAndNonce new_segment_keys = 6; } diff --git a/proto.lock b/proto.lock index 5b9694119..619cca074 100644 --- a/proto.lock +++ b/proto.lock @@ -606,6 +606,11 @@ "name": "default_path_cipher", "type": "encryption.CipherSuite" }, + { + "id": 5, + "name": "default_metadata_cipher", + "type": "encryption.CipherSuite" + }, { "id": 4, "name": "default_encryption_parameters", @@ -641,6 +646,11 @@ "name": "path_cipher", "type": "encryption.CipherSuite" }, + { + "id": 7, + "name": "metadata_cipher", + "type": "encryption.CipherSuite" + }, { "id": 6, "name": "encryption_parameters", @@ -2664,6 +2674,11 @@ "name": "encrypted_metadata_encrypted_key", "type": "bytes" }, + { + "id": 22, + "name": "clear_metadata", + "type": "bytes" + }, { "id": 11, "name": "fixed_segment_size", @@ -2784,6 +2799,11 @@ "name": "encrypted_metadata_encrypted_key", "type": "bytes" }, + { + "id": 16, + "name": "clear_metadata", + "type": "bytes" + }, { "id": 12, "name": "retention", @@ -2893,6 +2913,11 @@ "id": 4, "name": "encrypted_metadata_encrypted_key", "type": "bytes" + }, + { + "id": 16, + "name": "clear_metadata", + "type": "bytes" } ] }, @@ -3290,6 +3315,11 @@ "name": "encrypted_metadata", "type": "bytes" }, + { + "id": 14, + "name": "clear_metadata", + "type": "bytes" + }, { "id": 10, "name": "plain_size", @@ -3559,6 +3589,11 @@ "id": 6, "name": "encrypted_metadata_encrypted_key", "type": "bytes" + }, + { + "id": 16, + "name": "clear_metadata", + "type": "bytes" } ] }, @@ -5401,6 +5436,11 @@ "name": "new_encrypted_metadata_key", "type": "bytes" }, + { + "id": 16, + "name": "new_clear_metadata", + "type": "bytes" + }, { "id": 6, "name": "new_segment_keys", diff --git a/usermeta/usermeta.go b/usermeta/usermeta.go new file mode 100644 index 000000000..9f8b86e62 --- /dev/null +++ b/usermeta/usermeta.go @@ -0,0 +1,117 @@ +// Copyright (C) 2025 Storj Labs, Inc. +// See LICENSE for copying information. + +package usermeta + +import ( + "encoding/json" + "strings" + + "storj.io/common/pb" +) + +// UserMeta stores simple string key-value metadata for objects. +type UserMeta map[string]string + +// Marshal user metadata payload using the SerializableMeta structure. +func Marshal(meta UserMeta) ([]byte, error) { + m := &pb.SerializableMeta{ + UserDefined: meta, + } + + return pb.Marshal(m) +} + +// Marshal a JSON-encoded, deeply structured user metadata using the +// SerializableMeta structure. +func MarshalJSON(meta string) ([]byte, error) { + var dm DeepUserMeta + err := json.Unmarshal([]byte(meta), &dm) + if err != nil { + return nil, err + } + + m, err := dm.toUserMeta() + if err != nil { + return nil, err + } + + return Marshal(m) +} + +// Unmarshal user metadata payload using the SerializableMeta structure. +func Unmarshal(data []byte) (UserMeta, error) { + m := new(pb.SerializableMeta) + err := pb.Unmarshal(data, m) + if err != nil { + return nil, err + } + + return m.UserDefined, nil +} + +// UnmarshalJSON unmarshals the user metadata payload, converts it to a deeply +// structured metadata and returns it as a JSON string. +func UnmarshalJSON(data []byte) (string, error) { + m, err := Unmarshal(data) + if err != nil { + return "", err + } + + dm, err := m.toDeepUserMeta() + if err != nil { + return "", err + } + + j, err := json.Marshal(dm) + if err != nil { + return "", err + } + + return string(j), nil +} + +// Valid checks if the data is a valid user metadata payload. +func Valid(data []byte) bool { + _, err := Unmarshal(data) + return err == nil +} + +// DeepUserMeta stores an arbitrary deep metadata structures for +// objects. It can be converted from/to one-level UserMeta objects. +type DeepUserMeta map[string]interface{} + +func (m DeepUserMeta) toUserMeta() (UserMeta, error) { + meta := make(UserMeta) + + for k, v := range m { + if s, ok := v.(string); ok { + meta[k] = s + } else { + j, err := json.Marshal(v) + if err != nil { + return nil, err + } + meta["json:"+k] = string(j) + } + } + return meta, nil +} + +func (m UserMeta) toDeepUserMeta() (DeepUserMeta, error) { + meta := make(DeepUserMeta) + + for k, v := range m { + if strings.HasPrefix(k, "json:") { + var i interface{} + err := json.Unmarshal([]byte(v), &i) + if err != nil { + return nil, err + } + meta[k[5:]] = i + } else { + meta[k] = v + } + } + return meta, nil +} diff --git a/usermeta/usermeta_test.go b/usermeta/usermeta_test.go new file mode 100644 index 000000000..631115247 --- /dev/null +++ b/usermeta/usermeta_test.go @@ -0,0 +1,57 @@ +// Copyright (C) 2025 Storj Labs, Inc. +// See LICENSE for copying information. + +package usermeta + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUserMetaMarshal(t *testing.T) { + meta := UserMeta{ + "key1": "value1", + "key2": "value2", + } + + data, err := Marshal(meta) + require.NoError(t, err) + + meta2, err := Unmarshal(data) + require.NoError(t, err) + + require.Equal(t, meta, meta2) +} + +func TestUserMetaMarshalJSON(t *testing.T) { + meta := `{"foo":"bar"}` + + data, err := MarshalJSON(meta) + require.NoError(t, err) + + meta2, err := UnmarshalJSON(data) + require.NoError(t, err) + + require.Equal(t, meta, meta2) +} + +func TestDeepUserMeta(t *testing.T) { + deepMeta := DeepUserMeta{ + "s": "text", + "o": map[string]interface{}{ + "a": []interface{}{1.0, 2.0, 3.0}, + }, + } + + meta, err := deepMeta.toUserMeta() + require.NoError(t, err) + require.Equal(t, UserMeta{ + "s": "text", + "json:o": `{"a":[1,2,3]}`, + }, meta) + + deepMeta2, err := meta.toDeepUserMeta() + require.NoError(t, err) + require.Equal(t, deepMeta, deepMeta2) +}