From ebbdcb0297bec6dd30f19c10b643f72b70471d94 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 9 Apr 2025 08:45:43 +0200 Subject: [PATCH 1/4] feat: add support for mount options --- pkg/rclone/rclone.go | 97 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 14 deletions(-) diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index 6286e5ed..bb8d7f28 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -57,14 +57,62 @@ type MountRequest struct { MountOpt MountOpt `json:"mountOpt"` } +// VfsOpt is options for creating the vfs type VfsOpt struct { - CacheMode string `json:"cacheMode"` - DirCacheTime time.Duration `json:"dirCacheTime"` - ReadOnly bool `json:"readOnly"` + // CacheMode string `json:"cacheMode"` + // DirCacheTime time.Duration `json:"dirCacheTime"` + // ReadOnly bool `json:"readOnly"` + + NoSeek bool `json:",omitempty"` // don't allow seeking if set + NoChecksum bool `json:",omitempty"` // don't check checksums if set + ReadOnly bool `json:",omitempty"` // if set VFS is read only + NoModTime bool `json:",omitempty"` // don't read mod times for files + DirCacheTime time.Duration `json:",omitempty"` // how long to consider directory listing cache valid + Refresh bool `json:",omitempty"` // refreshes the directory listing recursively on start + PollInterval time.Duration `json:",omitempty"` + Umask int `json:",omitempty"` + UID uint32 `json:",omitempty"` + GID uint32 `json:",omitempty"` + DirPerms os.FileMode `json:",omitempty"` + FilePerms os.FileMode `json:",omitempty"` + ChunkSize int64 `json:",omitempty"` // if > 0 read files in chunks + ChunkSizeLimit int64 `json:",omitempty"` // if > ChunkSize double the chunk size after each chunk until reached + CacheMode string `json:",omitempty"` + CacheMaxAge time.Duration `json:",omitempty"` + CacheMaxSize int64 `json:",omitempty"` + CacheMinFreeSpace int64 `json:",omitempty"` + CachePollInterval time.Duration `json:",omitempty"` + CaseInsensitive bool `json:",omitempty"` + WriteWait time.Duration `json:",omitempty"` // time to wait for in-sequence write + ReadWait time.Duration `json:",omitempty"` // time to wait for in-sequence read + WriteBack time.Duration `json:",omitempty"` // time to wait before writing back dirty files + ReadAhead int64 `json:",omitempty"` // bytes to read ahead in cache mode "full" + UsedIsSize bool `json:",omitempty"` // if true, use the `rclone size` algorithm for Used size + FastFingerprint bool `json:",omitempty"` // if set use fast fingerprints + DiskSpaceTotalSize int64 `json:",omitempty"` } + type MountOpt struct { - AllowNonEmpty bool `json:"allowNonEmpty"` - AllowOther bool `json:"allowOther"` + // AllowNonEmpty bool `json:"allowNonEmpty"` + // AllowOther bool `json:"allowOther"` + + DebugFUSE bool `json:",omitempty"` + AllowNonEmpty bool `json:",omitempty"` + AllowRoot bool `json:",omitempty"` + AllowOther bool `json:",omitempty"` + DefaultPermissions bool `json:",omitempty"` + WritebackCache bool `json:",omitempty"` + DaemonWait time.Duration `json:",omitempty"` // time to wait for ready mount from daemon, maximum on Linux or constant on macOS/BSD + MaxReadAhead int64 `json:",omitempty"` + ExtraOptions []string `json:",omitempty"` + ExtraFlags []string `json:",omitempty"` + AttrTimeout time.Duration `json:",omitempty"` // how long the kernel caches attribute for + DeviceName string `json:",omitempty"` + VolumeName string `json:",omitempty"` + NoAppleDouble bool `json:",omitempty"` + NoAppleXattr bool `json:",omitempty"` + AsyncRead bool `json:",omitempty"` + CaseInsensitive string `json:",omitempty"` } type ConfigCreateRequest struct { Name string `json:"name"` @@ -121,19 +169,40 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa } klog.Infof("created config: %s", configName) + // VFS Mount parameters + vfsOpt := VfsOpt{} + vfsOptStr := parameters["vfsOpt"] + if vfsOptStr != "" { + err = json.Unmarshal([]byte(vfsOptStr), &vfsOpt) + if err != nil { + return fmt.Errorf("could not parse vfsOpt: %w", err) + } + } + if vfsOpt.CacheMode == "" { + vfsOpt.CacheMode = "writes" + } + if vfsOpt.DirCacheTime == 0 { + vfsOpt.DirCacheTime = 60 * time.Second + } + vfsOpt.ReadOnly = readOnly + // Mount parameters + mountOpt := MountOpt{} + mountOptStr := parameters["mountOpt"] + if mountOptStr != "" { + err = json.Unmarshal([]byte(mountOptStr), &mountOpt) + if err != nil { + return fmt.Errorf("could not parse mountOpt: %w", err) + } + } + mountOpt.AllowNonEmpty = true + mountOpt.AllowOther = true + remoteWithPath := fmt.Sprintf("%s:%s", configName, rcloneVolume.RemotePath) mountArgs := MountRequest{ Fs: remoteWithPath, MountPoint: targetPath, - VfsOpt: VfsOpt{ - CacheMode: "writes", - DirCacheTime: 60 * time.Second, - ReadOnly: readOnly, - }, - MountOpt: MountOpt{ - AllowNonEmpty: true, - AllowOther: true, - }, + VfsOpt: vfsOpt, + MountOpt: mountOpt, } // create target, os.Mkdirall is noop if it exists From 14961c4df4ea999be0f32b0bdf2e5e000880a931 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 9 Apr 2025 09:25:35 +0200 Subject: [PATCH 2/4] improve log --- Dockerfile | 2 +- pkg/rclone/rclone.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8bc25284..099a3df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG RCLONE_IMAGE_REPOSITORY="ghcr.io/swissdatasciencecenter/rclone" -ARG RCLONE_IMAGE_TAG="sha-316bdfc" +ARG RCLONE_IMAGE_TAG="sha-1a32af4" FROM ${RCLONE_IMAGE_REPOSITORY}:${RCLONE_IMAGE_TAG} AS rclone FROM golang:1.23.0-bookworm AS build diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index bb8d7f28..8e8ff399 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -210,11 +210,11 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa if err != nil { return err } - klog.Infof("executing mount command args=%v, targetpath=%s", mountArgs, targetPath) postBody, err = json.Marshal(mountArgs) if err != nil { return fmt.Errorf("mounting failed: couldn't create request body: %s", err) } + klog.Infof("executing mount command args=%v, targetpath=%s", string(postBody), targetPath) requestBody = bytes.NewBuffer(postBody) resp, err = http.Post(fmt.Sprintf("http://localhost:%d/mount/mount", r.port), "application/json", requestBody) if err != nil { From 438d4345de567913a0abdaba0ba3ba7c43d05556 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 9 Apr 2025 09:49:13 +0200 Subject: [PATCH 3/4] nicer code for defaults --- pkg/rclone/rclone.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index 8e8ff399..c396ff0f 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -58,11 +58,9 @@ type MountRequest struct { } // VfsOpt is options for creating the vfs +// +// Note that the `Daemon` option has been removed as it is not accepted for rc calls. type VfsOpt struct { - // CacheMode string `json:"cacheMode"` - // DirCacheTime time.Duration `json:"dirCacheTime"` - // ReadOnly bool `json:"readOnly"` - NoSeek bool `json:",omitempty"` // don't allow seeking if set NoChecksum bool `json:",omitempty"` // don't check checksums if set ReadOnly bool `json:",omitempty"` // if set VFS is read only @@ -92,10 +90,10 @@ type VfsOpt struct { DiskSpaceTotalSize int64 `json:",omitempty"` } +// Options for creating the mount +// +// Note that options not supported on Linux have been removed. type MountOpt struct { - // AllowNonEmpty bool `json:"allowNonEmpty"` - // AllowOther bool `json:"allowOther"` - DebugFUSE bool `json:",omitempty"` AllowNonEmpty bool `json:",omitempty"` AllowRoot bool `json:",omitempty"` @@ -114,6 +112,7 @@ type MountOpt struct { AsyncRead bool `json:",omitempty"` CaseInsensitive string `json:",omitempty"` } + type ConfigCreateRequest struct { Name string `json:"name"` Parameters map[string]string `json:"parameters"` @@ -170,7 +169,10 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa klog.Infof("created config: %s", configName) // VFS Mount parameters - vfsOpt := VfsOpt{} + vfsOpt := VfsOpt{ + CacheMode: "writes", + DirCacheTime: 60 * time.Second, + } vfsOptStr := parameters["vfsOpt"] if vfsOptStr != "" { err = json.Unmarshal([]byte(vfsOptStr), &vfsOpt) @@ -178,15 +180,13 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa return fmt.Errorf("could not parse vfsOpt: %w", err) } } - if vfsOpt.CacheMode == "" { - vfsOpt.CacheMode = "writes" - } - if vfsOpt.DirCacheTime == 0 { - vfsOpt.DirCacheTime = 60 * time.Second - } + // The `ReadOnly` option is specified in the PVC vfsOpt.ReadOnly = readOnly // Mount parameters - mountOpt := MountOpt{} + mountOpt := MountOpt{ + AllowNonEmpty: true, + AllowOther: true, + } mountOptStr := parameters["mountOpt"] if mountOptStr != "" { err = json.Unmarshal([]byte(mountOptStr), &mountOpt) @@ -194,8 +194,6 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa return fmt.Errorf("could not parse mountOpt: %w", err) } } - mountOpt.AllowNonEmpty = true - mountOpt.AllowOther = true remoteWithPath := fmt.Sprintf("%s:%s", configName, rcloneVolume.RemotePath) mountArgs := MountRequest{ From cb3a4e7f4bae9a4a31a2d67bb3e60cc7a31f8e41 Mon Sep 17 00:00:00 2001 From: Flora Thiebaut Date: Wed, 9 Apr 2025 09:57:18 +0200 Subject: [PATCH 4/4] improve logs --- pkg/rclone/rclone.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index c396ff0f..4cb2a8d5 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -152,7 +152,7 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa Parameters: params, Opt: map[string]interface{}{"obscure": true}, } - klog.Infof("executing create config command args=%v, targetpath=%s", configName, targetPath) + klog.Infof("executing create config command name=%s, storageType=%s", configName, configOpts.StorageType) postBody, err := json.Marshal(configOpts) if err != nil { return fmt.Errorf("mounting failed: couldn't create request body: %s", err) @@ -212,7 +212,7 @@ func (r *Rclone) Mount(ctx context.Context, rcloneVolume *RcloneVolume, targetPa if err != nil { return fmt.Errorf("mounting failed: couldn't create request body: %s", err) } - klog.Infof("executing mount command args=%v, targetpath=%s", string(postBody), targetPath) + klog.Infof("executing mount command args=%s", string(postBody)) requestBody = bytes.NewBuffer(postBody) resp, err = http.Post(fmt.Sprintf("http://localhost:%d/mount/mount", r.port), "application/json", requestBody) if err != nil {