diff --git a/proto/api/v1/instance_service.proto b/proto/api/v1/instance_service.proto index ebe9ed2f1d5bc..aef6bd3e820a6 100644 --- a/proto/api/v1/instance_service.proto +++ b/proto/api/v1/instance_service.proto @@ -140,6 +140,7 @@ message InstanceSetting { string region = 4; string bucket = 5; bool use_path_style = 6; + string custom_domain = 7; } // The S3 config. S3Config s3_config = 4; diff --git a/proto/gen/api/v1/instance_service.pb.go b/proto/gen/api/v1/instance_service.pb.go index c98eb3b88bae6..cb93febcc52d5 100644 --- a/proto/gen/api/v1/instance_service.pb.go +++ b/proto/gen/api/v1/instance_service.pb.go @@ -795,6 +795,7 @@ type InstanceSetting_StorageSetting_S3Config struct { Region string `protobuf:"bytes,4,opt,name=region,proto3" json:"region,omitempty"` Bucket string `protobuf:"bytes,5,opt,name=bucket,proto3" json:"bucket,omitempty"` UsePathStyle bool `protobuf:"varint,6,opt,name=use_path_style,json=usePathStyle,proto3" json:"use_path_style,omitempty"` + CustomDomain string `protobuf:"bytes,7,opt,name=custom_domain,json=customDomain,proto3" json:"custom_domain,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -871,6 +872,13 @@ func (x *InstanceSetting_StorageSetting_S3Config) GetUsePathStyle() bool { return false } +func (x *InstanceSetting_StorageSetting_S3Config) GetCustomDomain() string { + if x != nil { + return x.CustomDomain + } + return "" +} + var File_api_v1_instance_service_proto protoreflect.FileDescriptor const file_api_v1_instance_service_proto_rawDesc = "" + @@ -881,7 +889,7 @@ const file_api_v1_instance_service_proto_rawDesc = "" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x12\n" + "\x04mode\x18\x03 \x01(\tR\x04mode\x12!\n" + "\finstance_url\x18\x06 \x01(\tR\vinstanceUrl\"\x1b\n" + - "\x19GetInstanceProfileRequest\"\x99\x0f\n" + + "\x19GetInstanceProfileRequest\"\xbe\x0f\n" + "\x0fInstanceSetting\x12\x17\n" + "\x04name\x18\x01 \x01(\tB\x03\xe0A\bR\x04name\x12W\n" + "\x0fgeneral_setting\x18\x02 \x01(\v2,.memos.api.v1.InstanceSetting.GeneralSettingH\x00R\x0egeneralSetting\x12W\n" + @@ -899,19 +907,20 @@ const file_api_v1_instance_service_proto_rawDesc = "" + "\rCustomProfile\x12\x14\n" + "\x05title\x18\x01 \x01(\tR\x05title\x12 \n" + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x19\n" + - "\blogo_url\x18\x03 \x01(\tR\alogoUrl\x1a\xbc\x04\n" + + "\blogo_url\x18\x03 \x01(\tR\alogoUrl\x1a\xe1\x04\n" + "\x0eStorageSetting\x12[\n" + "\fstorage_type\x18\x01 \x01(\x0e28.memos.api.v1.InstanceSetting.StorageSetting.StorageTypeR\vstorageType\x12+\n" + "\x11filepath_template\x18\x02 \x01(\tR\x10filepathTemplate\x12/\n" + "\x14upload_size_limit_mb\x18\x03 \x01(\x03R\x11uploadSizeLimitMb\x12R\n" + - "\ts3_config\x18\x04 \x01(\v25.memos.api.v1.InstanceSetting.StorageSetting.S3ConfigR\bs3Config\x1a\xcc\x01\n" + + "\ts3_config\x18\x04 \x01(\v25.memos.api.v1.InstanceSetting.StorageSetting.S3ConfigR\bs3Config\x1a\xf1\x01\n" + "\bS3Config\x12\"\n" + "\raccess_key_id\x18\x01 \x01(\tR\vaccessKeyId\x12*\n" + "\x11access_key_secret\x18\x02 \x01(\tR\x0faccessKeySecret\x12\x1a\n" + "\bendpoint\x18\x03 \x01(\tR\bendpoint\x12\x16\n" + "\x06region\x18\x04 \x01(\tR\x06region\x12\x16\n" + "\x06bucket\x18\x05 \x01(\tR\x06bucket\x12$\n" + - "\x0euse_path_style\x18\x06 \x01(\bR\fusePathStyle\"L\n" + + "\x0euse_path_style\x18\x06 \x01(\bR\fusePathStyle\x12#\n" + + "\rcustom_domain\x18\a \x01(\tR\fcustomDomain\"L\n" + "\vStorageType\x12\x1c\n" + "\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\f\n" + "\bDATABASE\x10\x01\x12\t\n" + diff --git a/proto/gen/openapi.yaml b/proto/gen/openapi.yaml index b52c35b20f25e..9ec617a37d80c 100644 --- a/proto/gen/openapi.yaml +++ b/proto/gen/openapi.yaml @@ -2824,6 +2824,8 @@ components: type: string usePathStyle: type: boolean + customDomain: + type: string description: |- S3 configuration for cloud storage backend. Reference: https://developers.cloudflare.com/r2/examples/aws/aws-sdk-go/ diff --git a/proto/gen/store/instance_setting.pb.go b/proto/gen/store/instance_setting.pb.go index d368e626df57d..a99b3f77d33d4 100644 --- a/proto/gen/store/instance_setting.pb.go +++ b/proto/gen/store/instance_setting.pb.go @@ -563,6 +563,7 @@ type StorageS3Config struct { Region string `protobuf:"bytes,4,opt,name=region,proto3" json:"region,omitempty"` Bucket string `protobuf:"bytes,5,opt,name=bucket,proto3" json:"bucket,omitempty"` UsePathStyle bool `protobuf:"varint,6,opt,name=use_path_style,json=usePathStyle,proto3" json:"use_path_style,omitempty"` + CustomDomain string `protobuf:"bytes,7,opt,name=custom_domain,json=customDomain,proto3" json:"custom_domain,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -639,6 +640,13 @@ func (x *StorageS3Config) GetUsePathStyle() bool { return false } +func (x *StorageS3Config) GetCustomDomain() string { + if x != nil { + return x.CustomDomain + } + return "" +} + type InstanceMemoRelatedSetting struct { state protoimpl.MessageState `protogen:"open.v1"` // disallow_public_visibility disallows set memo as public visibility. @@ -758,14 +766,15 @@ const file_store_instance_setting_proto_rawDesc = "" + "\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\f\n" + "\bDATABASE\x10\x01\x12\t\n" + "\x05LOCAL\x10\x02\x12\x06\n" + - "\x02S3\x10\x03\"\xd3\x01\n" + + "\x02S3\x10\x03\"\xf8\x01\n" + "\x0fStorageS3Config\x12\"\n" + "\raccess_key_id\x18\x01 \x01(\tR\vaccessKeyId\x12*\n" + "\x11access_key_secret\x18\x02 \x01(\tR\x0faccessKeySecret\x12\x1a\n" + "\bendpoint\x18\x03 \x01(\tR\bendpoint\x12\x16\n" + "\x06region\x18\x04 \x01(\tR\x06region\x12\x16\n" + "\x06bucket\x18\x05 \x01(\tR\x06bucket\x12$\n" + - "\x0euse_path_style\x18\x06 \x01(\bR\fusePathStyle\"\x9c\x02\n" + + "\x0euse_path_style\x18\x06 \x01(\bR\fusePathStyle\x12#\n" + + "\rcustom_domain\x18\a \x01(\tR\fcustomDomain\"\x9c\x02\n" + "\x1aInstanceMemoRelatedSetting\x12<\n" + "\x1adisallow_public_visibility\x18\x01 \x01(\bR\x18disallowPublicVisibility\x127\n" + "\x18display_with_update_time\x18\x02 \x01(\bR\x15displayWithUpdateTime\x120\n" + diff --git a/proto/store/instance_setting.proto b/proto/store/instance_setting.proto index fcfcbdd60dfcf..13f151e593e0d 100644 --- a/proto/store/instance_setting.proto +++ b/proto/store/instance_setting.proto @@ -89,6 +89,7 @@ message StorageS3Config { string region = 4; string bucket = 5; bool use_path_style = 6; + string custom_domain = 7; } message InstanceMemoRelatedSetting { diff --git a/server/router/api/v1/attachment_service.go b/server/router/api/v1/attachment_service.go index 3226b9f42f563..a23986be4044f 100644 --- a/server/router/api/v1/attachment_service.go +++ b/server/router/api/v1/attachment_service.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "fmt" "io" + "net/url" "os" "path/filepath" "regexp" @@ -340,16 +341,29 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s filepathTemplate = filepath.Join(filepathTemplate, "{filename}") } filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename) - key, err := s3Client.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob)) + _, err = s3Client.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob)) if err != nil { return errors.Wrap(err, "Failed to upload via s3 client") } - presignURL, err := s3Client.PresignGetObject(ctx, key) - if err != nil { - return errors.Wrap(err, "Failed to presign via s3 client") + key := filepathTemplate + var referenceURL string + if s3Config.CustomDomain != "" { + domain := s3Config.CustomDomain + if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") { + domain = "https://" + domain + } + referenceURL, err = url.JoinPath(domain, key) + if err != nil { + return errors.Wrap(err, "Failed to join custom domain path") + } + } else { + referenceURL, err = s3Client.PresignGetObject(ctx, key) + if err != nil { + return errors.Wrap(err, "Failed to presign via s3 client") + } } - create.Reference = presignURL + create.Reference = referenceURL create.Blob = nil create.StorageType = storepb.AttachmentStorageType_S3 create.Payload = &storepb.AttachmentPayload{ diff --git a/server/router/api/v1/instance_service.go b/server/router/api/v1/instance_service.go index 82830112e40fa..f272cf054af49 100644 --- a/server/router/api/v1/instance_service.go +++ b/server/router/api/v1/instance_service.go @@ -10,6 +10,7 @@ import ( v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/server/runner/s3presign" "github.com/usememos/memos/store" ) @@ -99,6 +100,10 @@ func (s *APIV1Service) UpdateInstanceSetting(ctx context.Context, request *v1pb. return nil, status.Errorf(codes.Internal, "failed to upsert instance setting: %v", err) } + if instanceSetting.Key == storepb.InstanceSettingKey_STORAGE { + go s3presign.NewRunner(s.Store).CheckAndPresign(context.Background()) + } + return convertInstanceSettingFromStore(instanceSetting), nil } @@ -214,6 +219,7 @@ func convertInstanceStorageSettingFromStore(settingpb *storepb.InstanceStorageSe Region: settingpb.S3Config.Region, Bucket: settingpb.S3Config.Bucket, UsePathStyle: settingpb.S3Config.UsePathStyle, + CustomDomain: settingpb.S3Config.CustomDomain, } } return setting @@ -236,6 +242,7 @@ func convertInstanceStorageSettingToStore(setting *v1pb.InstanceSetting_StorageS Region: setting.S3Config.Region, Bucket: setting.S3Config.Bucket, UsePathStyle: setting.S3Config.UsePathStyle, + CustomDomain: setting.S3Config.CustomDomain, } } return settingpb diff --git a/server/runner/s3presign/runner.go b/server/runner/s3presign/runner.go index f8df43f25d68c..fe9a753514690 100644 --- a/server/runner/s3presign/runner.go +++ b/server/runner/s3presign/runner.go @@ -3,6 +3,8 @@ package s3presign import ( "context" "log/slog" + "net/url" + "strings" "time" "google.golang.org/protobuf/types/known/timestamppb" @@ -74,46 +76,86 @@ func (r *Runner) CheckAndPresign(ctx context.Context) { // Process batch of attachments presignCount := 0 + globalS3Config := instanceStorageSetting.GetS3Config() + for _, attachment := range attachments { s3ObjectPayload := attachment.Payload.GetS3Object() if s3ObjectPayload == nil { continue } - if s3ObjectPayload.LastPresignedTime != nil { - // Skip if the presigned URL is still valid for the next 4 days. - // The expiration time is set to 5 days. - if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) { - continue + var s3Config *storepb.StorageS3Config + if s3ObjectPayload.S3Config != nil { + if globalS3Config != nil { + s3Config = globalS3Config + } else { + s3Config = s3ObjectPayload.S3Config } + } else { + s3Config = globalS3Config } - s3Config := instanceStorageSetting.GetS3Config() - if s3ObjectPayload.S3Config != nil { - s3Config = s3ObjectPayload.S3Config - } if s3Config == nil { slog.Error("S3 config is not found") continue } - s3Client, err := s3.NewClient(ctx, s3Config) - if err != nil { - slog.Error("Failed to create S3 client", "error", err) - continue + if s3ObjectPayload.LastPresignedTime != nil { + // Skip if the presigned URL is still valid for the next 4 days. + // The expiration time is set to 5 days. + if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) { + // Check if the current reference URL matches the custom domain setting. + match := true + if s3Config.CustomDomain != "" { + domain := s3Config.CustomDomain + if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") { + domain = "https://" + domain + } + if !strings.HasPrefix(attachment.Reference, domain) { + match = false + } + } else { + // If custom domain is not set, the reference URL should be a presigned URL. + if !strings.Contains(attachment.Reference, "?") { + match = false + } + } + + if match { + continue + } + } } - presignURL, err := s3Client.PresignGetObject(ctx, s3ObjectPayload.Key) - if err != nil { - slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID) - continue + var referenceURL string + if s3Config.CustomDomain != "" { + domain := s3Config.CustomDomain + if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") { + domain = "https://" + domain + } + referenceURL, err = url.JoinPath(domain, s3ObjectPayload.Key) + if err != nil { + slog.Error("Failed to join custom domain path", "error", err, "attachmentID", attachment.ID) + continue + } + } else { + s3Client, err := s3.NewClient(ctx, s3Config) + if err != nil { + slog.Error("Failed to create S3 client", "error", err) + continue + } + referenceURL, err = s3Client.PresignGetObject(ctx, s3ObjectPayload.Key) + if err != nil { + slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID) + continue + } } s3ObjectPayload.S3Config = s3Config s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now()) if err := r.Store.UpdateAttachment(ctx, &store.UpdateAttachment{ ID: attachment.ID, - Reference: &presignURL, + Reference: &referenceURL, Payload: &storepb.AttachmentPayload{ Payload: &storepb.AttachmentPayload_S3Object_{ S3Object: s3ObjectPayload, diff --git a/web/src/components/Settings/StorageSection.tsx b/web/src/components/Settings/StorageSection.tsx index 56b325f7d78dc..f214cc9d03083 100644 --- a/web/src/components/Settings/StorageSection.tsx +++ b/web/src/components/Settings/StorageSection.tsx @@ -84,6 +84,7 @@ const StorageSection = () => { region: existingS3Config?.region ?? "", bucket: existingS3Config?.bucket ?? "", usePathStyle: existingS3Config?.usePathStyle ?? false, + customDomain: existingS3Config?.customDomain ?? "", ...s3Config, }; const update = create(InstanceSetting_StorageSettingSchema, { @@ -115,6 +116,10 @@ const StorageSection = () => { handlePartialS3ConfigChanged({ bucket: event.target.value }); }; + const handleS3ConfigCustomDomainChanged = async (event: React.FocusEvent) => { + handlePartialS3ConfigChanged({ customDomain: event.target.value }); + }; + const handleS3ConfigUsePathStyleChanged = (event: React.ChangeEvent) => { handlePartialS3ConfigChanged({ usePathStyle: event.target.checked, @@ -222,6 +227,10 @@ const StorageSection = () => { + + + +