Skip to content

Commit 20ea8e7

Browse files
sweatybridgesweatybridge
authored
feat(DEVWF-862): add vector and analytics buckets to storage config (#4559)
* feat: add vector and analytics buckets to storage config * chore: allow declaring analytics and vector buckets * feat: implement bucket upsert clients * chore: note on hosted platform
1 parent 2a3b4c9 commit 20ea8e7

File tree

7 files changed

+300
-16
lines changed

7 files changed

+300
-16
lines changed

‎internal/seed/buckets/buckets.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package buckets
33
import (
44
"context"
55
"fmt"
6+
"os"
67

78
"github.com/spf13/afero"
89
"github.com/supabase/cli/internal/storage/client"
@@ -29,5 +30,25 @@ func Run(ctx context.Context, projectRef string, interactive bool, fsys afero.Fs
2930
if err := api.UpsertBuckets(ctx, utils.Config.Storage.Buckets, filter); err != nil {
3031
return err
3132
}
33+
prune := func(name string) bool {
34+
label := fmt.Sprintf("Bucket %s not found in %s. Do you want to prune it?", utils.Bold(name), utils.Bold(utils.ConfigPath))
35+
shouldPrune, err := console.PromptYesNo(ctx, label, false)
36+
if err != nil {
37+
fmt.Fprintln(utils.GetDebugLogger(), err)
38+
}
39+
return shouldPrune
40+
}
41+
if utils.Config.Storage.AnalyticsBuckets.Enabled && len(projectRef) > 0 {
42+
fmt.Fprintln(os.Stderr, "Updating analytics buckets...")
43+
if err := api.UpsertAnalyticsBuckets(ctx, utils.Config.Storage.AnalyticsBuckets.Buckets, prune); err != nil {
44+
return err
45+
}
46+
}
47+
if utils.Config.Storage.VectorBuckets.Enabled && len(projectRef) > 0 {
48+
fmt.Fprintln(os.Stderr, "Updating vector buckets...")
49+
if err := api.UpsertVectorBuckets(ctx, utils.Config.Storage.VectorBuckets.Buckets, prune); err != nil {
50+
return err
51+
}
52+
}
3253
return api.UpsertObjects(ctx, utils.Config.Storage.Buckets, utils.NewRootFS(fsys))
3354
}

‎pkg/config/config.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,8 @@ func (a *auth) Clone() auth {
291291
func (s *storage) Clone() storage {
292292
copy := *s
293293
copy.Buckets = maps.Clone(s.Buckets)
294+
copy.AnalyticsBuckets.Buckets = maps.Clone(s.AnalyticsBuckets.Buckets)
295+
copy.VectorBuckets.Buckets = maps.Clone(s.VectorBuckets.Buckets)
294296
if s.ImageTransformation != nil {
295297
img := *s.ImageTransformation
296298
copy.ImageTransformation = &img
@@ -544,6 +546,15 @@ func (c *config) load(v *viper.Viper) error {
544546
}); err != nil {
545547
return errors.Errorf("failed to parse config: %w", err)
546548
}
549+
// Manually parse config to map
550+
c.Storage.AnalyticsBuckets.Buckets = map[string]struct{}{}
551+
for key := range v.GetStringMap("storage.analytics.buckets") {
552+
c.Storage.AnalyticsBuckets.Buckets[key] = struct{}{}
553+
}
554+
c.Storage.VectorBuckets.Buckets = map[string]struct{}{}
555+
for key := range v.GetStringMap("storage.vector.buckets") {
556+
c.Storage.VectorBuckets.Buckets[key] = struct{}{}
557+
}
547558
// Convert keys to upper case: https://github.com/spf13/viper/issues/1014
548559
secrets := make(SecretsConfig, len(c.EdgeRuntime.Secrets))
549560
for k, v := range c.EdgeRuntime.Secrets {

‎pkg/config/storage.go‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,29 @@ type (
1717
S3Protocol *s3Protocol `toml:"s3_protocol"`
1818
S3Credentials storageS3Credentials `toml:"-"`
1919
Buckets BucketConfig `toml:"buckets"`
20+
AnalyticsBuckets analyticsBuckets `toml:"analytics"`
21+
VectorBuckets vectorBuckets `toml:"vector"`
2022
}
2123

2224
imageTransformation struct {
2325
Enabled bool `toml:"enabled"`
2426
}
2527

28+
analyticsBuckets struct {
29+
Enabled bool `toml:"enabled"`
30+
MaxNamespaces uint `toml:"max_namespaces"`
31+
MaxTables uint `toml:"max_tables"`
32+
MaxCatalogs uint `toml:"max_catalogs"`
33+
Buckets map[string]struct{} `toml:"buckets"`
34+
}
35+
36+
vectorBuckets struct {
37+
Enabled bool `toml:"enabled"`
38+
MaxBuckets uint `toml:"max_buckets"`
39+
MaxIndexes uint `toml:"max_indexes"`
40+
Buckets map[string]struct{} `toml:"buckets"`
41+
}
42+
2643
s3Protocol struct {
2744
Enabled bool `toml:"enabled"`
2845
}
@@ -68,10 +85,43 @@ func (s *storage) ToUpdateStorageConfigBody() v1API.UpdateStorageConfigBody {
6885
}
6986
// When local config is not set, we assume platform defaults should not change
7087
if s.ImageTransformation != nil {
71-
body.Features.ImageTransformation.Enabled = s.ImageTransformation.Enabled
88+
body.Features.ImageTransformation = &struct {
89+
Enabled bool `json:"enabled"`
90+
}{
91+
Enabled: s.ImageTransformation.Enabled,
92+
}
93+
}
94+
// Disabling analytics and vector buckets means leaving platform values unchanged
95+
if s.AnalyticsBuckets.Enabled {
96+
body.Features.IcebergCatalog = &struct {
97+
Enabled bool `json:"enabled"`
98+
MaxCatalogs int `json:"maxCatalogs"`
99+
MaxNamespaces int `json:"maxNamespaces"`
100+
MaxTables int `json:"maxTables"`
101+
}{
102+
Enabled: true,
103+
MaxNamespaces: cast.UintToInt(s.AnalyticsBuckets.MaxNamespaces),
104+
MaxTables: cast.UintToInt(s.AnalyticsBuckets.MaxTables),
105+
MaxCatalogs: cast.UintToInt(s.AnalyticsBuckets.MaxCatalogs),
106+
}
107+
}
108+
if s.VectorBuckets.Enabled {
109+
body.Features.VectorBuckets = &struct {
110+
Enabled bool `json:"enabled"`
111+
MaxBuckets int `json:"maxBuckets"`
112+
MaxIndexes int `json:"maxIndexes"`
113+
}{
114+
Enabled: true,
115+
MaxBuckets: cast.UintToInt(s.VectorBuckets.MaxBuckets),
116+
MaxIndexes: cast.UintToInt(s.VectorBuckets.MaxIndexes),
117+
}
72118
}
73119
if s.S3Protocol != nil {
74-
body.Features.S3Protocol.Enabled = s.S3Protocol.Enabled
120+
body.Features.S3Protocol = &struct {
121+
Enabled bool `json:"enabled"`
122+
}{
123+
Enabled: s.S3Protocol.Enabled,
124+
}
75125
}
76126
return body
77127
}
@@ -83,6 +133,17 @@ func (s *storage) FromRemoteStorageConfig(remoteConfig v1API.StorageConfigRespon
83133
if s.ImageTransformation != nil {
84134
s.ImageTransformation.Enabled = remoteConfig.Features.ImageTransformation.Enabled
85135
}
136+
if s.AnalyticsBuckets.Enabled {
137+
s.AnalyticsBuckets.Enabled = remoteConfig.Features.IcebergCatalog.Enabled
138+
s.AnalyticsBuckets.MaxNamespaces = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxNamespaces)
139+
s.AnalyticsBuckets.MaxTables = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxTables)
140+
s.AnalyticsBuckets.MaxCatalogs = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxCatalogs)
141+
}
142+
if s.VectorBuckets.Enabled {
143+
s.VectorBuckets.Enabled = remoteConfig.Features.VectorBuckets.Enabled
144+
s.VectorBuckets.MaxBuckets = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxBuckets)
145+
s.VectorBuckets.MaxIndexes = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxIndexes)
146+
}
86147
if s.S3Protocol != nil {
87148
s.S3Protocol.Enabled = remoteConfig.Features.S3Protocol.Enabled
88149
}

‎pkg/config/templates/config.toml‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -105,20 +105,42 @@ enabled = true
105105
# The maximum file size allowed (e.g. "5MB", "500KB").
106106
file_size_limit = "50MiB"
107107

108-
# Image transformation API is available to Supabase Pro plan.
109-
# [storage.image_transformation]
110-
# enabled = true
111-
112-
# [storage.s3_protocol]
113-
# enabled = false
114-
115108
# Uncomment to configure local storage buckets
116109
# [storage.buckets.images]
117110
# public = false
118111
# file_size_limit = "50MiB"
119112
# allowed_mime_types = ["image/png", "image/jpeg"]
120113
# objects_path = "./images"
121114

115+
# Uncomment to allow connections via S3 compatible clients
116+
# [storage.s3_protocol]
117+
# enabled = true
118+
119+
# Image transformation API is available to Supabase Pro plan.
120+
# [storage.image_transformation]
121+
# enabled = true
122+
123+
# Store analytical data in S3 for running ETL jobs over Iceberg Catalog
124+
# This feature is only available on the hosted platform.
125+
[storage.analytics]
126+
enabled = false
127+
max_namespaces = 5
128+
max_tables = 10
129+
max_catalogs = 2
130+
131+
# Analytics Buckets is available to Supabase Pro plan.
132+
# [storage.analytics.buckets.my-warehouse]
133+
134+
# Store vector embeddings in S3 for large and durable datasets
135+
# This feature is only available on the hosted platform.
136+
[storage.vector]
137+
enabled = false
138+
max_buckets = 10
139+
max_indexes = 5
140+
141+
# Vector Buckets is available to Supabase Pro plan.
142+
# [storage.vector.buckets.documents-openai]
143+
122144
[auth]
123145
enabled = true
124146
# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used

‎pkg/config/testdata/config.toml‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -105,20 +105,40 @@ enabled = true
105105
# The maximum file size allowed (e.g. "5MB", "500KB").
106106
file_size_limit = "50MiB"
107107

108-
# Image transformation API is available to Supabase Pro plan.
109-
[storage.image_transformation]
110-
enabled = true
111-
112-
[storage.s3_protocol]
113-
enabled = true
114-
115108
# Uncomment to configure local storage buckets
116109
[storage.buckets.images]
117110
public = false
118111
file_size_limit = "50MiB"
119112
allowed_mime_types = ["image/png", "image/jpeg"]
120113
objects_path = "./images"
121114

115+
# Uncomment to allow connections via S3 compatible clients
116+
[storage.s3_protocol]
117+
enabled = true
118+
119+
# Image transformation API is available to Supabase Pro plan.
120+
[storage.image_transformation]
121+
enabled = true
122+
123+
# Store analytical data in S3 for running ETL jobs over Iceberg Catalog
124+
[storage.analytics]
125+
enabled = true
126+
max_namespaces = 5
127+
max_tables = 10
128+
max_catalogs = 2
129+
130+
# Analytics Buckets is available to Supabase Pro plan.
131+
[storage.analytics.buckets.my-warehouse]
132+
133+
# Store vector embeddings in S3 for large and durable datasets
134+
[storage.vector]
135+
enabled = true
136+
max_buckets = 10
137+
max_indexes = 5
138+
139+
# Vector Buckets is available to Supabase Pro plan.
140+
# [storage.vector.buckets.documents-openai]
141+
122142
[auth]
123143
enabled = true
124144
# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used

‎pkg/storage/analytics.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package storage
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"os"
8+
9+
"github.com/supabase/cli/pkg/fetcher"
10+
)
11+
12+
type AnalyticsBucketResponse struct {
13+
Id string `json:"id"` // "test"
14+
Name string `json:"name"` // "test"
15+
CreatedAt string `json:"created_at"` // "2023-10-13T17:48:58.491Z"
16+
UpdatedAt string `json:"updated_at"` // "2023-10-13T17:48:58.491Z"
17+
}
18+
19+
type CreateAnalyticsBucketRequest struct {
20+
BucketName string `json:"bucketName"`
21+
}
22+
23+
func (s *StorageAPI) UpsertAnalyticsBuckets(ctx context.Context, bucketConfig map[string]struct{}, filter ...func(string) bool) error {
24+
resp, err := s.Send(ctx, http.MethodGet, "/storage/v1/iceberg/bucket", nil)
25+
if err != nil {
26+
return err
27+
}
28+
buckets, err := fetcher.ParseJSON[[]AnalyticsBucketResponse](resp.Body)
29+
if err != nil {
30+
return err
31+
}
32+
var toDelete []string
33+
exists := make(map[string]struct{}, len(buckets))
34+
for _, b := range buckets {
35+
exists[b.Name] = struct{}{}
36+
if _, ok := bucketConfig[b.Name]; !ok {
37+
toDelete = append(toDelete, b.Name)
38+
}
39+
}
40+
for name := range bucketConfig {
41+
if _, ok := exists[name]; ok {
42+
fmt.Fprintln(os.Stderr, "Bucket already exists:", name)
43+
continue
44+
}
45+
fmt.Fprintln(os.Stderr, "Creating analytics bucket:", name)
46+
body := CreateAnalyticsBucketRequest{BucketName: name}
47+
if resp, err := s.Send(ctx, http.MethodPost, "/storage/v1/iceberg/bucket", body); err != nil {
48+
return err
49+
} else if err := resp.Body.Close(); err != nil {
50+
fmt.Fprintln(os.Stderr, err)
51+
}
52+
}
53+
OUTER:
54+
for _, name := range toDelete {
55+
for _, keep := range filter {
56+
if !keep(name) {
57+
continue OUTER
58+
}
59+
}
60+
fmt.Fprintln(os.Stderr, "Pruning analytics bucket:", name)
61+
if resp, err := s.Send(ctx, http.MethodDelete, "/storage/v1/iceberg/bucket/"+name, nil); err != nil {
62+
return err
63+
} else if err := resp.Body.Close(); err != nil {
64+
fmt.Fprintln(os.Stderr, err)
65+
}
66+
}
67+
return nil
68+
}

0 commit comments

Comments
 (0)