Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions proto/api/v1/instance_service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ message InstanceSetting {
string region = 4;
string bucket = 5;
bool use_path_style = 6;
string custom_domain = 7;
}
// The S3 config.
S3Config s3_config = 4;
Expand Down
17 changes: 13 additions & 4 deletions proto/gen/api/v1/instance_service.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions proto/gen/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2824,6 +2824,8 @@ components:
type: string
usePathStyle:
type: boolean
customDomain:
type: string
description: |-
S3 configuration for cloud storage backend.
Reference: https://developers.cloudflare.com/r2/examples/aws/aws-sdk-go/
Expand Down
13 changes: 11 additions & 2 deletions proto/gen/store/instance_setting.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions proto/store/instance_setting.proto
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ message StorageS3Config {
string region = 4;
string bucket = 5;
bool use_path_style = 6;
string custom_domain = 7;
}

message InstanceMemoRelatedSetting {
Expand Down
24 changes: 19 additions & 5 deletions server/router/api/v1/attachment_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/binary"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -340,16 +341,29 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s
filepathTemplate = filepath.Join(filepathTemplate, "{filename}")
}
filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename)
key, err := s3Client.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob))
_, err = s3Client.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob))
if err != nil {
return errors.Wrap(err, "Failed to upload via s3 client")
}
presignURL, err := s3Client.PresignGetObject(ctx, key)
if err != nil {
return errors.Wrap(err, "Failed to presign via s3 client")
key := filepathTemplate
var referenceURL string
if s3Config.CustomDomain != "" {
domain := s3Config.CustomDomain
if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") {
domain = "https://" + domain
}
referenceURL, err = url.JoinPath(domain, key)
if err != nil {
return errors.Wrap(err, "Failed to join custom domain path")
}
} else {
referenceURL, err = s3Client.PresignGetObject(ctx, key)
if err != nil {
return errors.Wrap(err, "Failed to presign via s3 client")
}
}

create.Reference = presignURL
create.Reference = referenceURL
create.Blob = nil
create.StorageType = storepb.AttachmentStorageType_S3
create.Payload = &storepb.AttachmentPayload{
Expand Down
7 changes: 7 additions & 0 deletions server/router/api/v1/instance_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/server/runner/s3presign"
"github.com/usememos/memos/store"
)

Expand Down Expand Up @@ -99,6 +100,10 @@ func (s *APIV1Service) UpdateInstanceSetting(ctx context.Context, request *v1pb.
return nil, status.Errorf(codes.Internal, "failed to upsert instance setting: %v", err)
}

if instanceSetting.Key == storepb.InstanceSettingKey_STORAGE {
go s3presign.NewRunner(s.Store).CheckAndPresign(context.Background())
}

return convertInstanceSettingFromStore(instanceSetting), nil
}

Expand Down Expand Up @@ -214,6 +219,7 @@ func convertInstanceStorageSettingFromStore(settingpb *storepb.InstanceStorageSe
Region: settingpb.S3Config.Region,
Bucket: settingpb.S3Config.Bucket,
UsePathStyle: settingpb.S3Config.UsePathStyle,
CustomDomain: settingpb.S3Config.CustomDomain,
}
}
return setting
Expand All @@ -236,6 +242,7 @@ func convertInstanceStorageSettingToStore(setting *v1pb.InstanceSetting_StorageS
Region: setting.S3Config.Region,
Bucket: setting.S3Config.Bucket,
UsePathStyle: setting.S3Config.UsePathStyle,
CustomDomain: setting.S3Config.CustomDomain,
}
}
return settingpb
Expand Down
78 changes: 60 additions & 18 deletions server/runner/s3presign/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package s3presign
import (
"context"
"log/slog"
"net/url"
"strings"
"time"

"google.golang.org/protobuf/types/known/timestamppb"
Expand Down Expand Up @@ -74,46 +76,86 @@ func (r *Runner) CheckAndPresign(ctx context.Context) {

// Process batch of attachments
presignCount := 0
globalS3Config := instanceStorageSetting.GetS3Config()

for _, attachment := range attachments {
s3ObjectPayload := attachment.Payload.GetS3Object()
if s3ObjectPayload == nil {
continue
}

if s3ObjectPayload.LastPresignedTime != nil {
// Skip if the presigned URL is still valid for the next 4 days.
// The expiration time is set to 5 days.
if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) {
continue
var s3Config *storepb.StorageS3Config
if s3ObjectPayload.S3Config != nil {
if globalS3Config != nil {
s3Config = globalS3Config
} else {
s3Config = s3ObjectPayload.S3Config
}
} else {
s3Config = globalS3Config
}

s3Config := instanceStorageSetting.GetS3Config()
if s3ObjectPayload.S3Config != nil {
s3Config = s3ObjectPayload.S3Config
}
if s3Config == nil {
slog.Error("S3 config is not found")
continue
}

s3Client, err := s3.NewClient(ctx, s3Config)
if err != nil {
slog.Error("Failed to create S3 client", "error", err)
continue
if s3ObjectPayload.LastPresignedTime != nil {
// Skip if the presigned URL is still valid for the next 4 days.
// The expiration time is set to 5 days.
if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) {
// Check if the current reference URL matches the custom domain setting.
match := true
if s3Config.CustomDomain != "" {
domain := s3Config.CustomDomain
if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") {
domain = "https://" + domain
}
if !strings.HasPrefix(attachment.Reference, domain) {
match = false
}
} else {
// If custom domain is not set, the reference URL should be a presigned URL.
if !strings.Contains(attachment.Reference, "?") {
match = false
}
}

if match {
continue
}
}
}

presignURL, err := s3Client.PresignGetObject(ctx, s3ObjectPayload.Key)
if err != nil {
slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID)
continue
var referenceURL string
if s3Config.CustomDomain != "" {
domain := s3Config.CustomDomain
if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") {
domain = "https://" + domain
}
referenceURL, err = url.JoinPath(domain, s3ObjectPayload.Key)
if err != nil {
slog.Error("Failed to join custom domain path", "error", err, "attachmentID", attachment.ID)
continue
}
} else {
s3Client, err := s3.NewClient(ctx, s3Config)
if err != nil {
slog.Error("Failed to create S3 client", "error", err)
continue
}
referenceURL, err = s3Client.PresignGetObject(ctx, s3ObjectPayload.Key)
if err != nil {
slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID)
continue
}
}

s3ObjectPayload.S3Config = s3Config
s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now())
if err := r.Store.UpdateAttachment(ctx, &store.UpdateAttachment{
ID: attachment.ID,
Reference: &presignURL,
Reference: &referenceURL,
Payload: &storepb.AttachmentPayload{
Payload: &storepb.AttachmentPayload_S3Object_{
S3Object: s3ObjectPayload,
Expand Down
9 changes: 9 additions & 0 deletions web/src/components/Settings/StorageSection.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ const StorageSection = () => {
region: existingS3Config?.region ?? "",
bucket: existingS3Config?.bucket ?? "",
usePathStyle: existingS3Config?.usePathStyle ?? false,
customDomain: existingS3Config?.customDomain ?? "",
...s3Config,
};
const update = create(InstanceSetting_StorageSettingSchema, {
Expand Down Expand Up @@ -115,6 +116,10 @@ const StorageSection = () => {
handlePartialS3ConfigChanged({ bucket: event.target.value });
};

const handleS3ConfigCustomDomainChanged = async (event: React.FocusEvent<HTMLInputElement>) => {
handlePartialS3ConfigChanged({ customDomain: event.target.value });
};

const handleS3ConfigUsePathStyleChanged = (event: React.ChangeEvent<HTMLInputElement>) => {
handlePartialS3ConfigChanged({
usePathStyle: event.target.checked,
Expand Down Expand Up @@ -222,6 +227,10 @@ const StorageSection = () => {
<Input className="w-64" value={instanceStorageSetting.s3Config?.bucket} onChange={handleS3ConfigBucketChanged} />
</SettingRow>

<SettingRow label="Custom Domain">
<Input className="w-64" value={instanceStorageSetting.s3Config?.customDomain} onChange={handleS3ConfigCustomDomainChanged} />
</SettingRow>

<SettingRow label="Use Path Style">
<Switch
checked={instanceStorageSetting.s3Config?.usePathStyle}
Expand Down
Loading