Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for unencrypted object metadata #16

Closed
wants to merge 4 commits into from
Closed

Add support for unencrypted object metadata #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c96c7f3
pb: add metadata ciphersuite fields to EncryptionAccess
gyim Jan 6, 2025
1da6bc7
pb: add clear_metadata field to Object and request structures
gyim Jan 7, 2025
64c7bd5
usermeta: add package to marshal/unmarshal user-defined object metadata
gyim Jan 7, 2025
825286e
usermeta: add support for deep metadata structures
gyim Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions encryption/path_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (

func newStore(key storj.Key, pathCipher storj.CipherSuite) *Store {
store := NewStore()
if err := store.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, key, pathCipher); err != nil {
if err := store.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, key, pathCipher, storj.EncNull); err != nil {
panic(err)
}
return store
Expand Down Expand Up @@ -128,7 +128,7 @@ func TestStoreEncryption_BucketRoot(t *testing.T) {
if !assert.NoError(t, err, errTag) {
continue
}
err = bucketStore.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, *bucketKey, cipher)
err = bucketStore.AddWithCipher("bucket", paths.Unencrypted{}, paths.Encrypted{}, *bucketKey, cipher, storj.EncNull)
if !assert.NoError(t, err, errTag) {
continue
}
Expand Down Expand Up @@ -187,7 +187,7 @@ func TestStoreEncryption_MultipleBases(t *testing.T) {
encPrefix, err := EncryptPath("bucket", prefix, cipher, rootStore)
require.NoError(t, err)

err = prefixStore.AddWithCipher("bucket", prefix, encPrefix, *prefixKey, cipher)
err = prefixStore.AddWithCipher("bucket", prefix, encPrefix, *prefixKey, cipher, storj.EncNull)
require.NoError(t, err)

path := paths.NewUnencrypted(rawPath)
Expand Down
61 changes: 38 additions & 23 deletions encryption/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ import (
// b1, u6/u7 => <{e8:u8}, [u7], <u6, e6, k6>>
// b2, u1 => <{}, [u1], <u1, e1', k1'>>
type Store struct {
roots map[string]*node
defaultKey *storj.Key
defaultPathCipher storj.CipherSuite
roots map[string]*node
defaultKey *storj.Key
defaultPathCipher storj.CipherSuite
defaultMetadataCipher storj.CipherSuite

// EncryptionBypass makes it so we can interoperate with
// the network without having encryption keys. paths will be encrypted but
Expand All @@ -51,9 +52,10 @@ func (s *Store) Clone() *Store {
}

clone := &Store{
roots: make(map[string]*node),
defaultPathCipher: s.defaultPathCipher,
EncryptionBypass: s.EncryptionBypass,
roots: make(map[string]*node),
defaultPathCipher: s.defaultPathCipher,
defaultMetadataCipher: s.defaultMetadataCipher,
EncryptionBypass: s.EncryptionBypass,
}

// Deep copy the defaultKey if it's not nil
Expand Down Expand Up @@ -120,11 +122,12 @@ func (n *node) clone() *node {

// Base represents a key with which to derive further keys at some encrypted/unencrypted path.
type Base struct {
Unencrypted paths.Unencrypted
Encrypted paths.Encrypted
Key storj.Key
PathCipher storj.CipherSuite
Default bool
Unencrypted paths.Unencrypted
Encrypted paths.Encrypted
Key storj.Key
PathCipher storj.CipherSuite
MetadataCipher storj.CipherSuite
Default bool
}

// clone returns a copy of the Base. The implementation can be simple because the
Expand Down Expand Up @@ -172,24 +175,35 @@ func (s *Store) GetDefaultPathCipher() storj.CipherSuite {
return s.defaultPathCipher
}

// SetDefaultMetadataCipher adds a default metadata cipher to be returned for any lookup that does not match a bucket.
func (s *Store) SetDefaultMetadataCipher(defaultMetadataCipher storj.CipherSuite) {
s.defaultMetadataCipher = defaultMetadataCipher
}

// GetDefaultMetadataCipher returns the default metadata cipher, or EncUnspecified if none has been set.
func (s *Store) GetDefaultMetadataCipher() storj.CipherSuite {
return s.defaultMetadataCipher
}

// Add creates a mapping from the unencrypted path to the encrypted path and key. It uses the current default cipher.
func (s *Store) Add(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key) error {
return s.AddWithCipher(bucket, unenc, enc, key, s.defaultPathCipher)
return s.AddWithCipher(bucket, unenc, enc, key, s.defaultPathCipher, s.defaultMetadataCipher)
}

// AddWithCipher creates a mapping from the unencrypted path to the encrypted path and key with the given cipher.
func (s *Store) AddWithCipher(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error {
func (s *Store) AddWithCipher(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error {
root, ok := s.roots[bucket]
if !ok {
root = newNode()
}

// Perform the addition starting at the root node.
if err := root.add(unenc.Iterator(), enc.Iterator(), &Base{
Unencrypted: unenc,
Encrypted: enc,
Key: key,
PathCipher: pathCipher,
Unencrypted: unenc,
Encrypted: enc,
Key: key,
PathCipher: pathCipher,
MetadataCipher: metadataCipher,
}); err != nil {
return err
}
Expand Down Expand Up @@ -289,9 +303,10 @@ func (s *Store) LookupEncrypted(bucket string, path paths.Encrypted) (

func (s *Store) defaultBase() *Base {
return &Base{
Key: *s.defaultKey,
PathCipher: s.defaultPathCipher,
Default: true,
Key: *s.defaultKey,
PathCipher: s.defaultPathCipher,
MetadataCipher: s.defaultMetadataCipher,
Default: true,
}
}

Expand Down Expand Up @@ -358,7 +373,7 @@ func (n *node) iterate(fn func(string, paths.Unencrypted, paths.Encrypted, storj
}

// IterateWithCipher executes the callback with every value that has been Added to the Store.
func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite) error) error {
func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite, storj.CipherSuite) error) error {
for bucket, root := range s.roots {
if err := root.iterateWithCipher(fn, bucket); err != nil {
return err
Expand All @@ -368,9 +383,9 @@ func (s *Store) IterateWithCipher(fn func(string, paths.Unencrypted, paths.Encry
}

// iterateWithCipher calls the callback if the node has a base, and recurses to its children.
func (n *node) iterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite) error, bucket string) error {
func (n *node) iterateWithCipher(fn func(string, paths.Unencrypted, paths.Encrypted, storj.Key, storj.CipherSuite, storj.CipherSuite) error, bucket string) error {
if n.base != nil {
err := fn(bucket, n.base.Unencrypted, n.base.Encrypted, n.base.Key, n.base.PathCipher)
err := fn(bucket, n.base.Unencrypted, n.base.Encrypted, n.base.Key, n.base.PathCipher, n.base.MetadataCipher)
if err != nil {
return err
}
Expand Down
81 changes: 44 additions & 37 deletions encryption/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,27 @@ func abortIfError(err error) {
}
}

func forAllCipherPairs(test func(c1 storj.CipherSuite, c2 storj.CipherSuite)) {
forAllCiphers(func(c1 storj.CipherSuite) {
forAllCiphers(func(c2 storj.CipherSuite) {
test(c1, c2)
})
})
}

func ExampleStore() {
s := NewStore()
ep := paths.NewEncrypted
up := paths.NewUnencrypted

// Add a fairly complicated tree to the store.
abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u6"), ep("e6"), toKey("k6"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b2", up("u1"), ep("e1'"), toKey("k1"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u6"), ep("e6"), toKey("k6"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b2", up("u1"), ep("e1'"), toKey("k1"), storj.EncAESGCM, storj.EncAESGCM))
abortIfError(s.AddWithCipher("b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), storj.EncAESGCM, storj.EncAESGCM))

// Look up some complicated queries by the unencrypted path.
printLookup(s.LookupUnencrypted("b1", up("u1")))
Expand Down Expand Up @@ -107,7 +115,7 @@ func ExampleStore_SetDefaultKey() {
ep := paths.NewEncrypted
up := paths.NewUnencrypted

abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM))
abortIfError(s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), storj.EncAESGCM, storj.EncAESGCM))

printLookup(s.LookupUnencrypted("b1", up("u1")))
printLookup(s.LookupUnencrypted("b1", up("u1/u2")))
Expand Down Expand Up @@ -145,15 +153,15 @@ func TestStoreErrors(t *testing.T) {
up := paths.NewUnencrypted

// Too many encrypted parts
require.Error(t, s.AddWithCipher("b1", up("u1"), ep("e1/e2/e3"), storj.Key{}, pathCipher))
require.Error(t, s.AddWithCipher("b1", up("u1"), ep("e1/e2/e3"), storj.Key{}, pathCipher, storj.EncNull))

// Too many unencrypted parts
require.Error(t, s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1"), storj.Key{}, pathCipher))
require.Error(t, s.AddWithCipher("b1", up("u1/u2/u3"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull))

// Mismatches
require.NoError(t, s.AddWithCipher("b1", up("u1"), ep("e1"), storj.Key{}, pathCipher))
require.Error(t, s.AddWithCipher("b1", up("u2"), ep("e1"), storj.Key{}, pathCipher))
require.Error(t, s.AddWithCipher("b1", up("u1"), ep("f1"), storj.Key{}, pathCipher))
require.NoError(t, s.AddWithCipher("b1", up("u1"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull))
require.Error(t, s.AddWithCipher("b1", up("u2"), ep("e1"), storj.Key{}, pathCipher, storj.EncNull))
require.Error(t, s.AddWithCipher("b1", up("u1"), ep("f1"), storj.Key{}, pathCipher, storj.EncNull))
}
}

Expand All @@ -166,9 +174,9 @@ func TestStoreErrorState(t *testing.T) {
revealed1, consumed1, base1 := s.LookupUnencrypted("b1", up("u1/u2"))

// Attempt to do an addition that fails.
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncNull))
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncAESGCM))
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncSecretBox))
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncNull, storj.EncNull))
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncAESGCM, storj.EncAESGCM))
require.Error(t, s.AddWithCipher("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{}, storj.EncSecretBox, storj.EncSecretBox))

// Ensure that we get the same results as before
revealed2, consumed2, base2 := s.LookupUnencrypted("b1", up("u1/u2"))
Expand All @@ -180,18 +188,15 @@ func TestStoreErrorState(t *testing.T) {

func TestStoreIterate(t *testing.T) {
type storeEntry struct {
bucket string
unenc paths.Unencrypted
enc paths.Encrypted
key storj.Key
pathCipher storj.CipherSuite
bucket string
unenc paths.Unencrypted
enc paths.Encrypted
key storj.Key
pathCipher storj.CipherSuite
metadataCipher storj.CipherSuite
}

for _, pathCipher := range []storj.CipherSuite{
storj.EncNull,
storj.EncAESGCM,
storj.EncSecretBox,
} {
forAllCipherPairs(func(pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) {
for _, bypass := range []bool{false, true} {
s := NewStore()
s.EncryptionBypass = bypass
Expand All @@ -200,27 +205,27 @@ func TestStoreIterate(t *testing.T) {
up := paths.NewUnencrypted

expected := map[storeEntry]struct{}{
{"b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), pathCipher}: {},
{"b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), pathCipher}: {},
{"b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), pathCipher}: {},
{"b1", up("u6"), ep("e6"), toKey("k6"), pathCipher}: {},
{"b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), pathCipher}: {},
{"b2", up("u1"), ep("e1'"), toKey("k1"), pathCipher}: {},
{"b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), pathCipher}: {},
{"b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"), pathCipher, metadataCipher}: {},
{"b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"), pathCipher, metadataCipher}: {},
{"b1", up("u1/u5"), ep("e1/e5"), toKey("k5"), pathCipher, metadataCipher}: {},
{"b1", up("u6"), ep("e6"), toKey("k6"), pathCipher, metadataCipher}: {},
{"b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"), pathCipher, metadataCipher}: {},
{"b2", up("u1"), ep("e1'"), toKey("k1"), pathCipher, metadataCipher}: {},
{"b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"), pathCipher, metadataCipher}: {},
}

for entry := range expected {
require.NoError(t, s.AddWithCipher(entry.bucket, entry.unenc, entry.enc, entry.key, entry.pathCipher))
require.NoError(t, s.AddWithCipher(entry.bucket, entry.unenc, entry.enc, entry.key, entry.pathCipher, entry.metadataCipher))
}

got := make(map[storeEntry]struct{})
require.NoError(t, s.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error {
got[storeEntry{bucket, unenc, enc, key, pathCipher}] = struct{}{}
require.NoError(t, s.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error {
got[storeEntry{bucket, unenc, enc, key, pathCipher, metadataCipher}] = struct{}{}
return nil
}))
require.Equal(t, expected, got)
}
}
})
}

func TestStoreEncryptionBypass(t *testing.T) {
Expand Down Expand Up @@ -248,6 +253,7 @@ func TestStoreClone(t *testing.T) {
store := NewStore()
store.SetDefaultKey(&defaultKey)
store.SetDefaultPathCipher(storj.EncAESGCM)
store.SetDefaultMetadataCipher(storj.EncAESGCM)
err := store.Add("bucket1", paths.NewUnencrypted("path1"), paths.NewEncrypted("encPath1"), pathKey)
require.NoError(t, err)

Expand All @@ -256,6 +262,7 @@ func TestStoreClone(t *testing.T) {
assert.Equal(t, store, clone)

assert.Equal(t, store.defaultPathCipher, clone.defaultPathCipher)
assert.Equal(t, store.defaultMetadataCipher, clone.defaultMetadataCipher)
assert.Equal(t, store.EncryptionBypass, clone.EncryptionBypass)

assert.NotSame(t, store.defaultKey, clone.defaultKey)
Expand Down
26 changes: 20 additions & 6 deletions grant/access.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ func (s *EncryptionAccess) SetDefaultPathCipher(defaultPathCipher storj.CipherSu
s.Store.SetDefaultPathCipher(defaultPathCipher)
}

// SetDefaultMetadataCipher sets which cipher suite to use by default for metadata.
func (s *EncryptionAccess) SetDefaultMetadataCipher(defaultMetadataCipher storj.CipherSuite) {
s.Store.SetDefaultMetadataCipher(defaultMetadataCipher)
}

// LimitTo limits the data in the encryption access only to the paths that would be
// allowed by the api key. Any errors that happen due to the consistency of the api
// key cause no keys to be stored.
Expand Down Expand Up @@ -231,7 +236,8 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store,

// create the new store that we'll load into and carry some necessary defaults
store := encryption.NewStore()
store.SetDefaultPathCipher(s.Store.GetDefaultPathCipher()) // keep default path cipher
store.SetDefaultPathCipher(s.Store.GetDefaultPathCipher()) // keep default path cipher
store.SetDefaultMetadataCipher(s.Store.GetDefaultMetadataCipher()) // keep default metadata cipher

// add the prefixes to the store, skipping any that fail for any reason
for _, prefix := range prefixes {
Expand All @@ -254,7 +260,7 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store,
continue // this should not happen given Decrypt succeeded, but whatever
}

if err := store.AddWithCipher(bucket, unencPath, encPath, *key, base.PathCipher); err != nil {
if err := store.AddWithCipher(bucket, unencPath, encPath, *key, base.PathCipher, base.MetadataCipher); err != nil {
continue
}
}
Expand All @@ -264,13 +270,14 @@ func (s *EncryptionAccess) limitTo(apiKey *macaroon.APIKey) (*encryption.Store,

func (s *EncryptionAccess) toProto() (*pb.EncryptionAccess, error) {
var storeEntries []*pb.EncryptionAccess_StoreEntry
err := s.Store.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite) error {
err := s.Store.IterateWithCipher(func(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key, pathCipher storj.CipherSuite, metadataCipher storj.CipherSuite) error {
storeEntries = append(storeEntries, &pb.EncryptionAccess_StoreEntry{
Bucket: []byte(bucket),
UnencryptedPath: []byte(unenc.Raw()),
EncryptedPath: []byte(enc.Raw()),
Key: key[:],
PathCipher: pb.CipherSuite(pathCipher),
MetadataCipher: pb.CipherSuite(metadataCipher),
})
return nil
})
Expand All @@ -284,9 +291,10 @@ func (s *EncryptionAccess) toProto() (*pb.EncryptionAccess, error) {
}

return &pb.EncryptionAccess{
DefaultKey: defaultKey,
StoreEntries: storeEntries,
DefaultPathCipher: pb.CipherSuite(s.Store.GetDefaultPathCipher()),
DefaultKey: defaultKey,
StoreEntries: storeEntries,
DefaultPathCipher: pb.CipherSuite(s.Store.GetDefaultPathCipher()),
DefaultMetadataCipher: pb.CipherSuite(s.Store.GetDefaultMetadataCipher()),
}, nil
}

Expand All @@ -308,6 +316,11 @@ func parseEncryptionAccessFromProto(p *pb.EncryptionAccess) (*EncryptionAccess,
access.SetDefaultPathCipher(storj.EncAESGCM)
}

access.SetDefaultMetadataCipher(storj.CipherSuite(p.DefaultMetadataCipher))
if p.DefaultMetadataCipher == pb.CipherSuite_ENC_UNSPECIFIED {
access.SetDefaultMetadataCipher(storj.EncAESGCM)
}

for _, entry := range p.StoreEntries {
if len(entry.Key) != len(storj.Key{}) {
return nil, errors.New("invalid key in encryption access entry")
Expand All @@ -321,6 +334,7 @@ func parseEncryptionAccessFromProto(p *pb.EncryptionAccess) (*EncryptionAccess,
paths.NewEncrypted(string(entry.EncryptedPath)),
key,
storj.CipherSuite(entry.PathCipher),
storj.CipherSuite(entry.MetadataCipher),
)
if err != nil {
return nil, fmt.Errorf("invalid encryption access entry: %w", err)
Expand Down
Loading