diff --git a/Dockerfile b/Dockerfile index 8bc25284..099a3df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG RCLONE_IMAGE_REPOSITORY="ghcr.io/swissdatasciencecenter/rclone" -ARG RCLONE_IMAGE_TAG="sha-316bdfc" +ARG RCLONE_IMAGE_TAG="sha-1a32af4" FROM ${RCLONE_IMAGE_REPOSITORY}:${RCLONE_IMAGE_TAG} AS rclone FROM golang:1.23.0-bookworm AS build diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index 6286e5ed..4cb2a8d5 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -57,15 +57,62 @@ type MountRequest struct { MountOpt MountOpt `json:"mountOpt"` } +// VfsOpt is options for creating the vfs +// +// Note that the `Daemon` option has been removed as it is not accepted for rc calls. type VfsOpt struct { - CacheMode string `json:"cacheMode"` - DirCacheTime time.Duration `json:"dirCacheTime"` - ReadOnly bool `json:"readOnly"` + NoSeek bool `json:",omitempty"` // don't allow seeking if set + NoChecksum bool `json:",omitempty"` // don't check checksums if set + ReadOnly bool `json:",omitempty"` // if set VFS is read only + NoModTime bool `json:",omitempty"` // don't read mod times for files + DirCacheTime time.Duration `json:",omitempty"` // how long to consider directory listing cache valid + Refresh bool `json:",omitempty"` // refreshes the directory listing recursively on start + PollInterval time.Duration `json:",omitempty"` + Umask int `json:",omitempty"` + UID uint32 `json:",omitempty"` + GID uint32 `json:",omitempty"` + DirPerms os.FileMode `json:",omitempty"` + FilePerms os.FileMode `json:",omitempty"` + ChunkSize int64 `json:",omitempty"` // if > 0 read files in chunks + ChunkSizeLimit int64 `json:",omitempty"` // if > ChunkSize double the chunk size after each chunk until reached + CacheMode string `json:",omitempty"` + CacheMaxAge time.Duration `json:",omitempty"` + CacheMaxSize int64 `json:",omitempty"` + CacheMinFreeSpace int64 `json:",omitempty"` + CachePollInterval time.Duration `json:",omitempty"` + CaseInsensitive bool `json:",omitempty"` + WriteWait time.Duration `json:",omitempty"` // time to wait for in-sequence write + ReadWait time.Duration `json:",omitempty"` // time to wait for in-sequence read + WriteBack time.Duration `json:",omitempty"` // time to wait before writing back dirty files + ReadAhead int64 `json:",omitempty"` // bytes to read ahead in cache mode "full" + UsedIsSize bool `json:",omitempty"` // if true, use the `rclone size` algorithm for Used size + FastFingerprint bool `json:",omitempty"` // if set use fast fingerprints + DiskSpaceTotalSize int64 `json:",omitempty"` } + +// Options for creating the mount +// +// Note that options not supported on Linux have been removed. type MountOpt struct { - AllowNonEmpty bool `json:"allowNonEmpty"` - AllowOther bool `json:"allowOther"` + DebugFUSE bool `json:",omitempty"` + AllowNonEmpty bool `json:",omitempty"` + AllowRoot bool `json:",omitempty"` + AllowOther bool `json:",omitempty"` + DefaultPermissions bool `json:",omitempty"` + WritebackCache bool `json:",omitempty"` + DaemonWait time.Duration `json:",omitempty"` // time to wait for ready mount from daemon, maximum on Linux or constant on macOS/BSD + MaxReadAhead int64 `json:",omitempty"` + ExtraOptions []string `json:",omitempty"` + ExtraFlags []string `json:",omitempty"` + AttrTimeout time.Duration `json:",omitempty"` // how long the kernel caches attribute for + DeviceName string `json:",omitempty"` + VolumeName string `json:",omitempty"` + NoAppleDouble bool `json:",omitempty"` + NoAppleXattr bool `json:",omitempty"` + AsyncRead bool `json:",omitempty"` + CaseInsensitive string `json:",omitempty"` } + type ConfigCreateRequest struct { Name string `json:"name"` Parameters map[string]string `json:"parameters"` @@ -105,7 +152,7 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa Parameters: params, Opt: map[string]interface{}{"obscure": true}, } - klog.Infof("executing create config command args=%v, targetpath=%s", configName, targetPath) + klog.Infof("executing create config command name=%s, storageType=%s", configName, configOpts.StorageType) postBody, err := json.Marshal(configOpts) if err != nil { return fmt.Errorf("mounting failed: couldn't create request body: %s", err) @@ -121,19 +168,39 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa } klog.Infof("created config: %s", configName) + // VFS Mount parameters + vfsOpt := VfsOpt{ + CacheMode: "writes", + DirCacheTime: 60 * time.Second, + } + vfsOptStr := parameters["vfsOpt"] + if vfsOptStr != "" { + err = json.Unmarshal([]byte(vfsOptStr), &vfsOpt) + if err != nil { + return fmt.Errorf("could not parse vfsOpt: %w", err) + } + } + // The `ReadOnly` option is specified in the PVC + vfsOpt.ReadOnly = readOnly + // Mount parameters + mountOpt := MountOpt{ + AllowNonEmpty: true, + AllowOther: true, + } + mountOptStr := parameters["mountOpt"] + if mountOptStr != "" { + err = json.Unmarshal([]byte(mountOptStr), &mountOpt) + if err != nil { + return fmt.Errorf("could not parse mountOpt: %w", err) + } + } + remoteWithPath := fmt.Sprintf("%s:%s", configName, rcloneVolume.RemotePath) mountArgs := MountRequest{ Fs: remoteWithPath, MountPoint: targetPath, - VfsOpt: VfsOpt{ - CacheMode: "writes", - DirCacheTime: 60 * time.Second, - ReadOnly: readOnly, - }, - MountOpt: MountOpt{ - AllowNonEmpty: true, - AllowOther: true, - }, + VfsOpt: vfsOpt, + MountOpt: mountOpt, } // create target, os.Mkdirall is noop if it exists @@ -141,11 +208,11 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa if err != nil { return err } - klog.Infof("executing mount command args=%v, targetpath=%s", mountArgs, targetPath) postBody, err = json.Marshal(mountArgs) if err != nil { return fmt.Errorf("mounting failed: couldn't create request body: %s", err) } + klog.Infof("executing mount command args=%s", string(postBody)) requestBody = bytes.NewBuffer(postBody) resp, err = http.Post(fmt.Sprintf("http://localhost:%d/mount/mount", r.port), "application/json", requestBody) if err != nil {