feat: support storage s3 protocol config (#4545)

Author: sweatybridge

internal/start/start.go

@@ -244,9 +244,10 @@ func run(ctx context.Context, fsys afero.Fs, excludedContainers []string, dbConf
 	}
 
 	var started []string
-	var isStorageEnabled = utils.Config.Storage.Enabled && !isContainerExcluded(utils.Config.Storage.Image, excluded)
-	var isImgProxyEnabled = utils.Config.Storage.ImageTransformation != nil &&
+	isStorageEnabled := utils.Config.Storage.Enabled && !isContainerExcluded(utils.Config.Storage.Image, excluded)
+	isImgProxyEnabled := utils.Config.Storage.ImageTransformation != nil &&
 		utils.Config.Storage.ImageTransformation.Enabled && !isContainerExcluded(utils.Config.Storage.ImgProxyImage, excluded)
+	isS3ProtocolEnabled := utils.Config.Storage.S3Protocol != nil && utils.Config.Storage.S3Protocol.Enabled
 	fmt.Fprintln(os.Stderr, "Starting containers...")
 
 	// Start Logflare
@@ -997,6 +998,7 @@ EOF
 					fmt.Sprintf("ENABLE_IMAGE_TRANSFORMATION=%t", isImgProxyEnabled),
 					fmt.Sprintf("IMGPROXY_URL=http://%s:5001", utils.ImgProxyId),
 					"TUS_URL_PATH=/storage/v1/upload/resumable",
+					fmt.Sprintf("S3_PROTOCOL_ENABLED=%t", isS3ProtocolEnabled),
 					"S3_PROTOCOL_ACCESS_KEY_ID=" + utils.Config.Storage.S3Credentials.AccessKeyId,
 					"S3_PROTOCOL_ACCESS_KEY_SECRET=" + utils.Config.Storage.S3Credentials.SecretAccessKey,
 					"S3_PROTOCOL_PREFIX=/storage/v1",

internal/status/status.go

@@ -84,7 +84,7 @@ func (c *CustomName) toValues(exclude ...string) map[string]string {
 		values[c.MailpitURL] = fmt.Sprintf("http://%s:%d", utils.Config.Hostname, utils.Config.Inbucket.Port)
 		values[c.InbucketURL] = fmt.Sprintf("http://%s:%d", utils.Config.Hostname, utils.Config.Inbucket.Port)
 	}
-	if storageEnabled {
+	if storageEnabled && utils.Config.Storage.S3Protocol != nil && utils.Config.Storage.S3Protocol.Enabled {
 		values[c.StorageS3URL] = utils.GetApiUrl("/storage/v1/s3")
 		values[c.StorageS3AccessKeyId] = utils.Config.Storage.S3Credentials.AccessKeyId
 		values[c.StorageS3SecretAccessKey] = utils.Config.Storage.S3Credentials.SecretAccessKey

pkg/config/config.go

@@ -295,6 +295,10 @@ func (s *storage) Clone() storage {
 		img := *s.ImageTransformation
 		copy.ImageTransformation = &img
 	}
+	if s.S3Protocol != nil {
+		s3 := *s.S3Protocol
+		copy.S3Protocol = &s3
+	}
 	return copy
 }
 

pkg/config/storage.go

@@ -14,6 +14,7 @@ type (
 		ImgProxyImage       string               `toml:"-"`
 		FileSizeLimit       sizeInBytes          `toml:"file_size_limit"`
 		ImageTransformation *imageTransformation `toml:"image_transformation"`
+		S3Protocol          *s3Protocol          `toml:"s3_protocol"`
 		S3Credentials       storageS3Credentials `toml:"-"`
 		Buckets             BucketConfig         `toml:"buckets"`
 	}
@@ -22,6 +23,10 @@ type (
 		Enabled bool `toml:"enabled"`
 	}
 
+	s3Protocol struct {
+		Enabled bool `toml:"enabled"`
+	}
+
 	storageS3Credentials struct {
 		AccessKeyId     string `toml:"-"`
 		SecretAccessKey string `toml:"-"`
@@ -41,10 +46,7 @@ type (
 func (s *storage) ToUpdateStorageConfigBody() v1API.UpdateStorageConfigBody {
 	body := v1API.UpdateStorageConfigBody{
 		FileSizeLimit: cast.Ptr(int64(s.FileSizeLimit)),
-	}
-	// When local config is not set, we assume platform defaults should not change
-	if s.ImageTransformation != nil {
-		body.Features = &struct {
+		Features: &struct {
 			IcebergCatalog *struct {
 				Enabled       bool `json:"enabled"`
 				MaxCatalogs   int  `json:"maxCatalogs"`
@@ -62,9 +64,15 @@ func (s *storage) ToUpdateStorageConfigBody() v1API.UpdateStorageConfigBody {
 				MaxBuckets int  `json:"maxBuckets"`
 				MaxIndexes int  `json:"maxIndexes"`
 			} `json:"vectorBuckets,omitempty"`
-		}{}
+		}{},
+	}
+	// When local config is not set, we assume platform defaults should not change
+	if s.ImageTransformation != nil {
 		body.Features.ImageTransformation.Enabled = s.ImageTransformation.Enabled
 	}
+	if s.S3Protocol != nil {
+		body.Features.S3Protocol.Enabled = s.S3Protocol.Enabled
+	}
 	return body
 }
 
@@ -74,14 +82,13 @@ func (s *storage) FromRemoteStorageConfig(remoteConfig v1API.StorageConfigRespon
 	if s.ImageTransformation != nil {
 		s.ImageTransformation.Enabled = remoteConfig.Features.ImageTransformation.Enabled
 	}
+	if s.S3Protocol != nil {
+		s.S3Protocol.Enabled = remoteConfig.Features.S3Protocol.Enabled
+	}
 }
 
 func (s *storage) DiffWithRemote(remoteConfig v1API.StorageConfigResponse) ([]byte, error) {
 	copy := s.Clone()
-	if s.ImageTransformation != nil {
-		img := *s.ImageTransformation
-		copy.ImageTransformation = &img
-	}
 	// Convert the config values into easily comparable remoteConfig values
 	currentValue, err := ToTomlBytes(copy)
 	if err != nil {

pkg/config/templates/config.toml

@@ -109,6 +109,9 @@ file_size_limit = "50MiB"
 # [storage.image_transformation]
 # enabled = true
 
+# [storage.s3_protocol]
+# enabled = false
+
 # Uncomment to configure local storage buckets
 # [storage.buckets.images]
 # public = false

pkg/config/testdata/config.toml

@@ -109,6 +109,9 @@ file_size_limit = "50MiB"
 [storage.image_transformation]
 enabled = true
 
+[storage.s3_protocol]
+enabled = true
+
 # Uncomment to configure local storage buckets
 [storage.buckets.images]
 public = false

pkg/config/updater_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/h2non/gock"
 	"github.com/oapi-codegen/nullable"
+	openapi_types "github.com/oapi-codegen/runtime/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1API "github.com/supabase/cli/pkg/api"
@@ -243,6 +244,7 @@ func TestUpdateStorageConfig(t *testing.T) {
 			}{},
 		}
 		mockStorage.Features.ImageTransformation.Enabled = true
+		mockStorage.Features.S3Protocol.Enabled = true
 		gock.New(server).
 			Get("/v1/projects/test-project/config/storage").
 			Reply(http.StatusOK).
@@ -313,11 +315,18 @@ func TestUpdateRemoteConfig(t *testing.T) {
 			JSON(v1API.PostgresConfigResponse{
 				MaxConnections: cast.Ptr(cast.UintToInt(100)),
 			})
+		// Network config
+		gock.New(server).
+			Get("/v1/projects/test-project/network-restrictions").
+			Reply(http.StatusOK).
+			JSON(v1API.V1GetNetworkRestrictionsResponse{})
 		// Auth config
 		gock.New(server).
 			Get("/v1/projects/test-project/config/auth").
 			Reply(http.StatusOK).
-			JSON(v1API.AuthConfigResponse{})
+			JSON(v1API.AuthConfigResponse{
+				SmtpAdminEmail: nullable.NewNullableWithValue(openapi_types.Email("[email protected]")),
+			})
 		gock.New(server).
 			Patch("/v1/projects/test-project/config/auth").
 			Reply(http.StatusOK)
@@ -357,6 +366,9 @@ func TestUpdateRemoteConfig(t *testing.T) {
 				ImageTransformation: &imageTransformation{
 					Enabled: true,
 				},
+				S3Protocol: &s3Protocol{
+					Enabled: true,
+				},
 			},
 			Experimental: experimental{
 				Webhooks: &webhooks{