From 878f5ca6e1129290c3247435980a73643acc628f Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 13:28:14 +0530 Subject: [PATCH 01/22] Add Stat and Open interfaces --- vfs.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vfs.go b/vfs.go index 56775a28..59865df6 100644 --- a/vfs.go +++ b/vfs.go @@ -3,6 +3,7 @@ package vfs import ( "fmt" "io" + "io/fs" "regexp" "time" @@ -155,6 +156,10 @@ type Location interface { // // URI's for locations must always end with a slash. URI() string + + // Open opens the named file at this location. + // This implements the fs.FS interface from io/fs. + Open(name string) (fs.File, error) } // File represents a file on a file system. A File may or may not actually exist on the file system. @@ -240,6 +245,10 @@ type File interface { // URI returns the fully qualified absolute URI for the File. IE, s3://bucket/some/path/to/file.txt URI() string + + // Stat returns a fs.FileInfo describing the file. + // This implements the fs.File interface from io/fs. + Stat() (fs.FileInfo, error) } // Options are structs that contain various options specific to the file system From ce6bcec14b00b67f2a8541dd5279dbbf0128f371 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 13:29:22 +0530 Subject: [PATCH 02/22] Add error wrapper for stat method --- utils/errors.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/errors.go b/utils/errors.go index 5ac8f2ed..746c0635 100644 --- a/utils/errors.go +++ b/utils/errors.go @@ -66,3 +66,8 @@ func WrapMoveToLocationError(err error) error { func WrapMoveToFileError(err error) error { return fmt.Errorf("moveToFile error: %w", err) } + +// WrapStatError returns a wrapped Stat error +func WrapStatError(err error) error { + return fmt.Errorf("stat error: %w", err) +} From da487672dca986c7a35488bf19dd58425ae39b41 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 13:30:13 +0530 Subject: [PATCH 03/22] Add implementations of Stat and Open into mem backend --- backend/mem/file.go | 58 +++++++++++++++++++++++++++++++++++++++++ backend/mem/location.go | 31 ++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/backend/mem/file.go b/backend/mem/file.go index 336096ed..6f1499e3 100644 --- a/backend/mem/file.go +++ b/backend/mem/file.go @@ -549,3 +549,61 @@ func (f *File) Name() string { func (f *File) URI() string { return utils.GetFileURI(f) } + +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (fs.FileInfo, error) { + if exists, err := f.Exists(); !exists { + if err != nil { + return nil, utils.WrapStatError(err) + } + return nil, fs.ErrNotExist + } + + size, err := f.Size() + if err != nil { + return nil, utils.WrapStatError(err) + } + + lastMod, err := f.LastModified() + if err != nil { + return nil, utils.WrapStatError(err) + } + + return &memFileInfo{ + name: f.Name(), + size: int64(size), + modTime: *lastMod, + }, nil +} + +// memFileInfo implements fs.FileInfo interface for mem file system +type memFileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *memFileInfo) Name() string { + return fi.name +} + +func (fi *memFileInfo) Size() int64 { + return fi.size +} + +func (fi *memFileInfo) Mode() fs.FileMode { + return 0644 // Default permission +} + +func (fi *memFileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi *memFileInfo) IsDir() bool { + return false +} + +func (fi *memFileInfo) Sys() interface{} { + return nil +} diff --git a/backend/mem/location.go b/backend/mem/location.go index 4db90f9c..f73325c3 100644 --- a/backend/mem/location.go +++ b/backend/mem/location.go @@ -3,6 +3,7 @@ package mem import ( "errors" "fmt" + "io/fs" "os" "path" "regexp" @@ -232,3 +233,33 @@ func (l *Location) DeleteFile(relFilePath string, _ ...options.DeleteOption) err func (l *Location) URI() string { return utils.GetLocationURI(l) } + +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (fs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + // Return the file which already implements fs.File (Read, Close, Stat) + return vfsFile, nil +} From c594061f4d317e1cbdc52496043936236b12bde8 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 14:43:27 +0530 Subject: [PATCH 04/22] Add mock implementation of Stat method --- mocks/File.go | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/mocks/File.go b/mocks/File.go index cd49eeee..caffbda8 100644 --- a/mocks/File.go +++ b/mocks/File.go @@ -9,6 +9,7 @@ import ( time "time" vfs "github.com/c2fo/vfs/v7" + fs "io/fs" ) // File is an autogenerated mock type for the File type @@ -798,6 +799,63 @@ func (_c *File_String_Call) RunAndReturn(run func() string) *File_String_Call { return _c } +// Stat provides a mock function with no fields +func (_m *File) Stat() (fs.FileInfo, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Stat") + } + + var r0 fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func() (fs.FileInfo, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() fs.FileInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// File_Stat_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stat' +type File_Stat_Call struct { + *mock.Call +} + +// Stat is a helper method to define mock.On call +func (_e *File_Expecter) Stat() *File_Stat_Call { + return &File_Stat_Call{Call: _e.mock.On("Stat")} +} + +func (_c *File_Stat_Call) Run(run func()) *File_Stat_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *File_Stat_Call) Return(_a0 fs.FileInfo, _a1 error) *File_Stat_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *File_Stat_Call) RunAndReturn(run func() (fs.FileInfo, error)) *File_Stat_Call { + _c.Call.Return(run) + return _c +} + // Touch provides a mock function with no fields func (_m *File) Touch() error { ret := _m.Called() From c98a85e1dd9d1e8b9ef00351ecd8b23bbc2e1f7f Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 14:48:02 +0530 Subject: [PATCH 05/22] Add mock implementation for Open method at Location --- mocks/Location.go | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/mocks/Location.go b/mocks/Location.go index cb207e81..55ca1a79 100644 --- a/mocks/Location.go +++ b/mocks/Location.go @@ -11,6 +11,7 @@ import ( regexp "regexp" vfs "github.com/c2fo/vfs/v7" + fs "io/fs" ) // Location is an autogenerated mock type for the Location type @@ -764,6 +765,64 @@ func (_c *Location_Volume_Call) RunAndReturn(run func() string) *Location_Volume return _c } +// Open provides a mock function with given fields: name +func (_m *Location) Open(name string) (fs.File, error) { + ret := _m.Called(name) + + if len(ret) == 0 { + panic("no return value specified for Open") + } + + var r0 fs.File + var r1 error + if rf, ok := ret.Get(0).(func(string) (fs.File, error)); ok { + return rf(name) + } + if rf, ok := ret.Get(0).(func(string) fs.File); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fs.File) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Location_Open_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Open' +type Location_Open_Call struct { + *mock.Call +} + +// Open is a helper method to define mock.On call +// - name string +func (_e *Location_Expecter) Open(name interface{}) *Location_Open_Call { + return &Location_Open_Call{Call: _e.mock.On("Open", name)} +} + +func (_c *Location_Open_Call) Run(run func(name string)) *Location_Open_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *Location_Open_Call) Return(_a0 fs.File, _a1 error) *Location_Open_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Location_Open_Call) RunAndReturn(run func(string) (fs.File, error)) *Location_Open_Call { + _c.Call.Return(run) + return _c +} + // NewLocation creates a new instance of Location. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewLocation(t interface { From 957cc9c2c9458d9abe73bd4f4bf6532ae2226150 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 14:48:39 +0530 Subject: [PATCH 06/22] Add Stat and Open implementations for os backend --- backend/os/file.go | 11 +++++++++++ backend/os/location.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/backend/os/file.go b/backend/os/file.go index 39f74f6b..e49d1e35 100644 --- a/backend/os/file.go +++ b/backend/os/file.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path" "path/filepath" @@ -404,6 +405,16 @@ func (f *File) openFile() (*os.File, error) { return file, nil } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (fs.FileInfo, error) { + info, err := os.Stat(osFilePath(f)) + if err != nil { + return nil, utils.WrapStatError(err) + } + return info, nil +} + func openOSFile(filePath string) (*os.File, error) { // Ensure the path exists before opening the file, NoOp if dir already exists. var fileMode os.FileMode = 0666 diff --git a/backend/os/location.go b/backend/os/location.go index d3d545c4..3cc4b104 100644 --- a/backend/os/location.go +++ b/backend/os/location.go @@ -2,6 +2,7 @@ package os import ( "errors" + "io/fs" "os" "path" "path/filepath" @@ -214,6 +215,43 @@ func (l *Location) FileSystem() vfs.FileSystem { return l.fileSystem } +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (fs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + // Get the underlying os.File + osFile := vfsFile.(*File) + internalFile, err := osFile.getInternalFile() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Return the os.File which already implements fs.File + return internalFile, nil +} + func osLocationPath(l vfs.Location) string { if runtime.GOOS == "windows" { return l.Authority().String() + filepath.FromSlash(l.Path()) From f54b9307780b6213fe1ca439d64fa4e886553dc2 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 14:54:09 +0530 Subject: [PATCH 07/22] Add implementations for azure backend --- backend/azure/file.go | 60 +++++++++++++++++++++++++++++++++++++++ backend/azure/location.go | 30 ++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/backend/azure/file.go b/backend/azure/file.go index 281f3074..67a83e81 100644 --- a/backend/azure/file.go +++ b/backend/azure/file.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path" "strings" @@ -66,6 +67,65 @@ func (f *File) Close() error { return nil } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (fs.FileInfo, error) { + exists, err := f.Exists() + if err != nil { + return nil, utils.WrapStatError(err) + } + if !exists { + return nil, fs.ErrNotExist + } + + size, err := f.Size() + if err != nil { + return nil, utils.WrapStatError(err) + } + + lastMod, err := f.LastModified() + if err != nil { + return nil, utils.WrapStatError(err) + } + + return &azureFileInfo{ + name: f.Name(), + size: int64(size), + modTime: *lastMod, + }, nil +} + +// azureFileInfo implements fs.FileInfo for Azure blobs +type azureFileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *azureFileInfo) Name() string { + return fi.name +} + +func (fi *azureFileInfo) Size() int64 { + return fi.size +} + +func (fi *azureFileInfo) Mode() fs.FileMode { + return 0644 // Default permission for files +} + +func (fi *azureFileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi *azureFileInfo) IsDir() bool { + return false // Azure blobs are always files, not directories +} + +func (fi *azureFileInfo) Sys() interface{} { + return nil +} + // Read implements the io.Reader interface. For this to work with Azure Blob Storage, a temporary local copy of // the file is created and read operations are performed against that. The temp file is closed and flushed to Azure // when f.Close() is called. diff --git a/backend/azure/location.go b/backend/azure/location.go index 68fff2a1..c82fecd5 100644 --- a/backend/azure/location.go +++ b/backend/azure/location.go @@ -2,6 +2,7 @@ package azure import ( "errors" + "io/fs" "path" "regexp" "strings" @@ -205,6 +206,35 @@ func (l *Location) NewFile(relFilePath string, opts ...options.NewFileOption) (v }, nil } +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (fs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + return vfsFile, nil +} + // DeleteFile deletes the file at the given path, relative to the current location. func (l *Location) DeleteFile(relFilePath string, opts ...options.DeleteOption) error { file, err := l.NewFile(utils.RemoveLeadingSlash(relFilePath)) From 9323f4728dbab714aa70b7abb7663e592ff4acfc Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 15:59:06 +0530 Subject: [PATCH 08/22] Add implementation for ftp --- backend/ftp/file.go | 49 ++++++++++++++++++++++++++++++++++++++++- backend/ftp/location.go | 30 +++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/backend/ftp/file.go b/backend/ftp/file.go index 380fea36..e209599c 100644 --- a/backend/ftp/file.go +++ b/backend/ftp/file.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + stdfs "io/fs" "os" "path" "strconv" @@ -79,6 +80,52 @@ func (f *File) stat(ctx context.Context) (*_ftp.Entry, error) { } } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (stdfs.FileInfo, error) { + entry, err := f.stat(context.TODO()) + if err != nil { + return nil, utils.WrapStatError(err) + } + + return &ftpFileInfo{ + name: f.Name(), + size: int64(entry.Size), + modTime: entry.Time, + }, nil +} + +// ftpFileInfo implements fs.FileInfo for FTP files +type ftpFileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *ftpFileInfo) Name() string { + return fi.name +} + +func (fi *ftpFileInfo) Size() int64 { + return fi.size +} + +func (fi *ftpFileInfo) Mode() stdfs.FileMode { + return 0644 // Default permission for files +} + +func (fi *ftpFileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi *ftpFileInfo) IsDir() bool { + return false // FTP files represented by File struct are always files, not directories +} + +func (fi *ftpFileInfo) Sys() interface{} { + return nil +} + // Name returns the path portion of the file's path property. IE: "file.txt" of "ftp://someuser@host.com/some/path/to/file.txt func (f *File) Name() string { return path.Base(f.path) @@ -109,7 +156,7 @@ func (f *File) Exists() (bool, error) { // Returns error if unable to touch File. func (f *File) Touch() error { exists, err := f.Exists() - if err != nil { + if (err != nil) { return utils.WrapTouchError(err) } diff --git a/backend/ftp/location.go b/backend/ftp/location.go index 1729865e..9a392fd9 100644 --- a/backend/ftp/location.go +++ b/backend/ftp/location.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + stdfs "io/fs" "path" "regexp" "strings" @@ -265,3 +266,32 @@ func (l *Location) URI() string { func (l *Location) String() string { return l.URI() } + +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (stdfs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: stdfs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: stdfs.ErrNotExist} + } + + return vfsFile, nil +} From b3647393c2fa7a307ad7bb891fd3934f041a2aa2 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 15:59:51 +0530 Subject: [PATCH 09/22] Add implementation for google storage --- backend/gs/file.go | 49 ++++++++++++++++++++++++++++++++++++++++++ backend/gs/location.go | 30 ++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/backend/gs/file.go b/backend/gs/file.go index 09b7ac36..0ae6ed30 100644 --- a/backend/gs/file.go +++ b/backend/gs/file.go @@ -13,6 +13,8 @@ import ( "cloud.google.com/go/storage" "google.golang.org/api/iterator" + "io/fs" + "github.com/c2fo/vfs/v7" "github.com/c2fo/vfs/v7/backend" "github.com/c2fo/vfs/v7/options" @@ -663,6 +665,53 @@ func (f *File) Size() (uint64, error) { return uint64(attr.Size), nil } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (fs.FileInfo, error) { + // Get file attributes from Google Cloud Storage + attrs, err := f.getObjectAttrs() + if err != nil { + return nil, utils.WrapStatError(err) + } + + return &gsFileInfo{ + name: f.Name(), + size: attrs.Size, + modTime: attrs.Updated, + }, nil +} + +// gsFileInfo implements fs.FileInfo for Google Cloud Storage files +type gsFileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *gsFileInfo) Name() string { + return fi.name +} + +func (fi *gsFileInfo) Size() int64 { + return fi.size +} + +func (fi *gsFileInfo) Mode() fs.FileMode { + return 0644 // Default permission for files +} + +func (fi *gsFileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi *gsFileInfo) IsDir() bool { + return false // GS files represented by File struct are always files, not directories +} + +func (fi *gsFileInfo) Sys() interface{} { + return nil +} + // Path returns full path with leading slash of the GCS file key. func (f *File) Path() string { return f.key diff --git a/backend/gs/location.go b/backend/gs/location.go index d4698a44..13001440 100644 --- a/backend/gs/location.go +++ b/backend/gs/location.go @@ -2,6 +2,7 @@ package gs import ( "errors" + "io/fs" "path" "regexp" "strings" @@ -225,6 +226,35 @@ func (l *Location) URI() string { return utils.GetLocationURI(l) } +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (fs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + return vfsFile, nil +} + // getBucketHandle returns cached Bucket struct for file func (l *Location) getBucketHandle() (BucketHandleWrapper, error) { if l.bucketHandle != nil { From 15d9a12dc82ac4c520e1ae40e88fbb53df264c3c Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 16:00:10 +0530 Subject: [PATCH 10/22] Add implementation for s3 --- backend/s3/file.go | 48 ++++++++++++++++++++++++++++++++++++++++++ backend/s3/location.go | 32 +++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/backend/s3/file.go b/backend/s3/file.go index 179dfcf9..842608c0 100644 --- a/backend/s3/file.go +++ b/backend/s3/file.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + stdfs "io/fs" "net/url" "os" "path" @@ -93,6 +94,53 @@ func (f *File) Size() (uint64, error) { return uint64(*head.ContentLength), nil } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (stdfs.FileInfo, error) { + // Get file metadata from S3 + headOutput, err := f.getHeadObject() + if err != nil { + return nil, utils.WrapStatError(err) + } + + return &s3FileInfo{ + name: f.Name(), + size: *headOutput.ContentLength, + modTime: *headOutput.LastModified, + }, nil +} + +// s3FileInfo implements fs.FileInfo for S3 files +type s3FileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *s3FileInfo) Name() string { + return fi.name +} + +func (fi *s3FileInfo) Size() int64 { + return fi.size +} + +func (fi *s3FileInfo) Mode() stdfs.FileMode { + return 0644 // Default permission for files +} + +func (fi *s3FileInfo) ModTime() time.Time { + return fi.modTime +} + +func (fi *s3FileInfo) IsDir() bool { + return false // S3 files represented by File struct are always files, not directories +} + +func (fi *s3FileInfo) Sys() interface{} { + return nil +} + // Location returns a vfs.Location at the location of the object. IE: if file is at // s3://bucket/here/is/the/file.txt the location points to s3://bucket/here/is/the/ func (f *File) Location() vfs.Location { diff --git a/backend/s3/location.go b/backend/s3/location.go index eb10be7c..ff6f68cf 100644 --- a/backend/s3/location.go +++ b/backend/s3/location.go @@ -3,6 +3,7 @@ package s3 import ( "context" "errors" + stdfs "io/fs" "path" "regexp" "strings" @@ -207,8 +208,37 @@ func (l *Location) String() string { return l.URI() } +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (stdfs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: stdfs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &stdfs.PathError{Op: "open", Path: name, Err: stdfs.ErrNotExist} + } + + return vfsFile, nil +} + /* - Private helpers +Private helpers */ func (l *Location) fullLocationList(input *s3.ListObjectsInput, prefix string) ([]string, error) { From 7cf35537d0cfa31424c3b4a8a8c7aa1bc8170189 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 16:00:26 +0530 Subject: [PATCH 11/22] Add implementation for sftp --- backend/sftp/file.go | 21 +++++++++++++++++++++ backend/sftp/location.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/backend/sftp/file.go b/backend/sftp/file.go index b673e23b..c0b46c28 100644 --- a/backend/sftp/file.go +++ b/backend/sftp/file.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path" "time" @@ -129,6 +130,26 @@ func (f *File) Size() (uint64, error) { return uint64(userinfo.Size()), nil } +// Stat returns a fs.FileInfo describing the file. +// This implements the fs.File interface from io/fs. +func (f *File) Stat() (fs.FileInfo, error) { + client, err := f.Location().FileSystem().(*FileSystem).Client(f.Location().Authority()) + if err != nil { + return nil, utils.WrapStatError(err) + } + + // Start timer once action is completed + defer f.Location().FileSystem().(*FileSystem).connTimerStart() + + fileInfo, err := client.Stat(f.Path()) + if err != nil { + return nil, utils.WrapStatError(err) + } + + // The SFTP client's FileInfo already implements fs.FileInfo, so we can just return it + return fileInfo, nil +} + // Location returns a vfs.Location at the location of the file. IE: if file is at // sftp://someuser@host.com/here/is/the/file.txt the location points to sftp://someuser@host.com/here/is/the/ func (f *File) Location() vfs.Location { diff --git a/backend/sftp/location.go b/backend/sftp/location.go index a90667a4..430fdc10 100644 --- a/backend/sftp/location.go +++ b/backend/sftp/location.go @@ -2,6 +2,7 @@ package sftp import ( "errors" + "io/fs" "os" "path" "regexp" @@ -250,3 +251,32 @@ func (l *Location) URI() string { func (l *Location) String() string { return l.URI() } + +// Open opens the named file at this location. +// This implements the fs.FS interface from io/fs. +func (l *Location) Open(name string) (fs.File, error) { + // fs.FS expects paths with no leading slash + name = strings.TrimPrefix(name, "/") + + // For io/fs compliance, we need to validate that it doesn't contain "." or ".." elements + if name == "." || name == ".." || strings.Contains(name, "/.") || strings.Contains(name, "./") { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + + // Create a standard vfs file using NewFile + vfsFile, err := l.NewFile(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + + // Check if the file exists, as fs.FS.Open requires the file to exist + exists, err := vfsFile.Exists() + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if !exists { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + return vfsFile, nil +} From bdfd8606763777c592768de42c755a857fd40038 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 16:02:28 +0530 Subject: [PATCH 12/22] Update changelog Fixes #163 --- CHANGELOG.md | 1 + backend/ftp/file.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 464b693a..8ca58786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Update vfs to implement io/fs's fs.FS interface fixes #163 ## [v7.4.1] - 2025-05-05 ### Security diff --git a/backend/ftp/file.go b/backend/ftp/file.go index e209599c..a06ff89c 100644 --- a/backend/ftp/file.go +++ b/backend/ftp/file.go @@ -156,7 +156,7 @@ func (f *File) Exists() (bool, error) { // Returns error if unable to touch File. func (f *File) Touch() error { exists, err := f.Exists() - if (err != nil) { + if err != nil { return utils.WrapTouchError(err) } From c28e0965838cc88d80299bcca8b4e4d7f307fd3c Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 16:29:17 +0530 Subject: [PATCH 13/22] Add tests for gs file --- backend/gs/file_test.go | 231 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/backend/gs/file_test.go b/backend/gs/file_test.go index d409673e..4154a52f 100644 --- a/backend/gs/file_test.go +++ b/backend/gs/file_test.go @@ -539,6 +539,237 @@ func (ts *fileTestSuite) TestMoveAndCopyBuffered() { } } +func (ts *fileTestSuite) TestSize() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: "text/plain", + ContentEncoding: "utf8", + }, + Content: []byte(contents), + }, + }, + ) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + size, err := file.Size() + ts.NoError(err, "Size() should not return an error for existing file") + ts.Equal(uint64(len(contents)), size, "Size should match the content length") +} + +func (ts *fileTestSuite) TestSizeError() { + bucketName := "bucki" + objectName := "nonexistent.txt" + server := fakestorage.NewServer(Objects{}) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + _, err = file.Size() + ts.Error(err, "Size() should return an error for non-existent file") +} + +func (ts *fileTestSuite) TestLastModified() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: "text/plain", + ContentEncoding: "utf8", + }, + Content: []byte(contents), + }, + }, + ) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + lastMod, err := file.LastModified() + ts.NoError(err, "LastModified() should not return an error for existing file") + ts.NotNil(lastMod, "LastModified should return a non-nil time") +} + +func (ts *fileTestSuite) TestLastModifiedError() { + bucketName := "bucki" + objectName := "nonexistent.txt" + server := fakestorage.NewServer(Objects{}) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + _, err = file.LastModified() + ts.Error(err, "LastModified() should return an error for non-existent file") +} + +func (ts *fileTestSuite) TestStat() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: "text/plain", + ContentEncoding: "utf8", + }, + Content: []byte(contents), + }, + }, + ) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + fileInfo, err := file.Stat() + ts.NoError(err, "Stat() should not return an error for existing file") + ts.NotNil(fileInfo, "FileInfo should not be nil") + ts.Equal("file.txt", fileInfo.Name(), "FileInfo name should match file name") + ts.Equal(int64(len(contents)), fileInfo.Size(), "FileInfo size should match content length") + ts.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") + ts.NotNil(fileInfo.ModTime(), "ModTime should not be nil") + ts.Equal(0644, int(fileInfo.Mode()), "Mode should be 0644") + ts.Nil(fileInfo.Sys(), "Sys should return nil") +} + +func (ts *fileTestSuite) TestStatError() { + bucketName := "bucki" + objectName := "nonexistent.txt" + server := fakestorage.NewServer(Objects{}) + defer server.Stop() + fs := NewFileSystem(WithClient(server.Client())) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + _, err = file.Stat() + ts.Error(err, "Stat() should return an error for non-existent file") +} + +func (ts *fileTestSuite) TestTouchExistingFileWithVersioning() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: "text/plain", + ContentEncoding: "utf8", + }, + Content: []byte(contents), + }, + }, + ) + defer server.Stop() + client := server.Client() + + // Enable versioning on the bucket manually + ctx := context.Background() + bucket := client.Bucket(bucketName) + _, err := bucket.Update(ctx, storage.BucketAttrsToUpdate{ + VersioningEnabled: true, + }) + ts.NoError(err, "Setting versioning should not error") + + fs := NewFileSystem(WithClient(client)) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + // This should use the updateLastModifiedByMoving path since versioning is enabled + err = file.Touch() + ts.NoError(err, "Touch() should not return an error for existing file with versioning") + + // Check the file still exists and is accessible + exists, err := file.Exists() + ts.NoError(err, "Exists() should not return an error") + ts.True(exists, "File should still exist after Touch with versioning") +} + +func (ts *fileTestSuite) TestTouchExistingFileWithoutVersioning() { + contents := "hello world!" + bucketName := "bucki" + objectName := "some/path/file.txt" + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucketName, + Name: objectName, + ContentType: "text/plain", + ContentEncoding: "utf8", + }, + Content: []byte(contents), + }, + }, + ) + defer server.Stop() + client := server.Client() + fs := NewFileSystem(WithClient(client)) + + file, err := fs.NewFile(bucketName, "/"+objectName) + ts.Require().NoError(err, "Shouldn't fail creating new file") + + // This should use the updateLastModifiedByAttrUpdate path since versioning is not enabled + err = file.Touch() + ts.NoError(err, "Touch() should not return an error for existing file without versioning") + + // Check the file still exists and is accessible + exists, err := file.Exists() + ts.NoError(err, "Exists() should not return an error") + ts.True(exists, "File should still exist after Touch without versioning") +} + +func (ts *fileTestSuite) TestNameAndPath() { + fs := NewFileSystem() + objectName := "some/path/file.txt" + + file, err := fs.NewFile("bucket", "/"+objectName) + ts.NoError(err, "Shouldn't fail creating new file") + + ts.Equal("file.txt", file.Name(), "Name should be just the filename") + ts.Equal("/some/path/file.txt", file.Path(), "Path should be the full path") +} + +func (ts *fileTestSuite) TestURI() { + fs := NewFileSystem() + objectName := "some/path/file.txt" + + file, err := fs.NewFile("bucket", "/"+objectName) + ts.NoError(err, "Shouldn't fail creating new file") + + ts.Equal("gs://bucket/some/path/file.txt", file.URI(), "URI should be correctly formatted") + ts.Equal("gs://bucket/some/path/file.txt", file.String(), "String() should return URI") +} + func TestFile(t *testing.T) { suite.Run(t, new(fileTestSuite)) } From 2fc5a9b2b7c8cb2209e58174b71b47b4df2ead2e Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 18:57:04 +0530 Subject: [PATCH 14/22] Add tests for Stat and Open in mem backend --- backend/mem/file_test.go | 28 +++++++++++++++++++ backend/mem/location_test.go | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/backend/mem/file_test.go b/backend/mem/file_test.go index 68f9efe1..84ecec34 100644 --- a/backend/mem/file_test.go +++ b/backend/mem/file_test.go @@ -758,6 +758,34 @@ func (s *memFileTest) TestFileNewWrite() { s.Equal("hello world", string(data)) } +// TestStat tests the Stat method returns correct file information +func (s *memFileTest) TestStat() { + // Write some content to the file + expectedContent := "test content for stat" + _, err := s.testFile.Write([]byte(expectedContent)) + s.NoError(err, "write error not expected") + s.NoError(s.testFile.Close(), "close error not expected") + + // Get file info via Stat() + fileInfo, err := s.testFile.Stat() + s.NoError(err, "stat error not expected") + s.NotNil(fileInfo, "FileInfo should not be nil") + + // Check file info properties + s.Equal("test.txt", fileInfo.Name(), "FileInfo name should match file name") + s.Equal(int64(len(expectedContent)), fileInfo.Size(), "FileInfo size should match content length") + s.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") + s.NotNil(fileInfo.ModTime(), "ModTime should not be nil") + s.Equal(0644, int(fileInfo.Mode()), "Mode should be 0644") + + // Test Stat on non-existent file + nonExistentFile, err := s.fileSystem.NewFile("", "/non-existent-file.txt") + s.NoError(err, "error creating reference to non-existent file") + _, err = nonExistentFile.Stat() + s.Error(err, "error expected when calling Stat on non-existent file") + s.ErrorIs(err, fs.ErrNotExist, "error should be fs.ErrNotExist") +} + func TestMemFile(t *testing.T) { suite.Run(t, new(memFileTest)) _ = os.Remove("test_files/new.txt") diff --git a/backend/mem/location_test.go b/backend/mem/location_test.go index e4461487..a10458fb 100644 --- a/backend/mem/location_test.go +++ b/backend/mem/location_test.go @@ -1,7 +1,9 @@ package mem import ( + "errors" "io" + "io/fs" "path" "regexp" "testing" @@ -342,6 +344,58 @@ func (s *memLocationTest) TestWriteExistingFile() { s.Equal("hello world", string(data)) } +// TestOpen tests the Open method in the Location implementation +func (s *memLocationTest) TestOpen() { + // Setup test files + testContent := "hello world" + testFileName := "open_test_file.txt" + testFilePath := "/test_files/" + testFileName + + // Create a file and write content + file, err := s.fileSystem.NewFile("", testFilePath) + s.NoError(err, "error not expected when creating a new file") + _, err = file.Write([]byte(testContent)) + s.NoError(err, "write error not expected") + s.NoError(file.Close(), "close error not expected") + + // Get the file's location + loc := file.Location() + + // Test Opening the file + opened, err := loc.Open(testFileName) + s.NoError(err, "error not expected when opening existing file") + s.NotNil(opened, "opened file should not be nil") + + // Read the content to verify + data := make([]byte, len(testContent)) + n, err := opened.Read(data) + s.NoError(err, "read error not expected") + s.Equal(len(testContent), n, "should read all content") + s.Equal(testContent, string(data), "content should match") + + // Test opening non-existent file + _, err = loc.Open("non-existent-file.txt") + s.Error(err, "error expected when opening non-existent file") + var pathErr *fs.PathError + s.True(errors.As(err, &pathErr), "error should be a fs.PathError") + s.ErrorIs(pathErr.Err, fs.ErrNotExist, "underlying error should be fs.ErrNotExist") + + // Test opening with path traversal attempts (should be rejected) + _, err = loc.Open("../outside.txt") + s.Error(err, "error expected when path contains traversal") + s.True(errors.As(err, &pathErr), "error should be a fs.PathError") + s.ErrorIs(pathErr.Err, fs.ErrInvalid, "underlying error should be fs.ErrInvalid") + + _, err = loc.Open("./file.txt") + s.Error(err, "error expected when path contains ./") + + _, err = loc.Open(".") + s.Error(err, "error expected when path is .") + + _, err = loc.Open("..") + s.Error(err, "error expected when path is ..") +} + func TestMemLocation(t *testing.T) { suite.Run(t, new(memLocationTest)) } From 83f68de61e57daef0e483fe4c8161eb5af705b2a Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 19:00:47 +0530 Subject: [PATCH 15/22] Add Open test for gs backend --- backend/gs/location_test.go | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/backend/gs/location_test.go b/backend/gs/location_test.go index 594dfa7e..c8b7d70c 100644 --- a/backend/gs/location_test.go +++ b/backend/gs/location_test.go @@ -323,6 +323,66 @@ func (lt *locationTestSuite) TestDeleteFile() { }) } +func (lt *locationTestSuite) TestOpen() { + // Setup test values + bucket := "test-open-bucket" + locPath := "/test-dir/" + fileName := "test-file.txt" + fileContent := "Hello, this is a test file content." + + // Create test server with fake objects + server := fakestorage.NewServer( + Objects{ + fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: bucket, + Name: "test-dir/test-file.txt", + }, + Content: []byte(fileContent), + }, + }, + ) + defer server.Stop() + + fs := NewFileSystem(WithClient(server.Client())) + + // Create location + loc, err := fs.NewLocation(bucket, locPath) + lt.NoError(err, "Creating location shouldn't return an error") + + // Test opening an existing file + file, err := loc.Open(fileName) + lt.NoError(err, "Opening an existing file should not return an error") + lt.NotNil(file, "Opened file should not be nil") + + // Read content to verify + data := make([]byte, len(fileContent)) + n, err := file.Read(data) + lt.NoError(err, "Reading from opened file should not error") + lt.Equal(len(fileContent), n, "Should read all content") + lt.Equal(fileContent, string(data), "File content should match") + + // Test opening a non-existent file + _, err = loc.Open("non-existent-file.txt") + lt.Error(err, "Opening a non-existent file should return an error") + lt.Contains(err.Error(), "ErrNotExist", "Error should indicate file does not exist") + + // Test opening with path traversal + _, err = loc.Open("../outside.txt") + lt.Error(err, "Opening a file with path traversal should return an error") + lt.Contains(err.Error(), "ErrInvalid", "Error should indicate invalid path") + + // Test opening with dot paths + _, err = loc.Open(".") + lt.Error(err, "Opening '.' should return an error") + + _, err = loc.Open("..") + lt.Error(err, "Opening '..' should return an error") + + _, err = loc.Open("./file.txt") + lt.Error(err, "Opening path with './' should return an error") +} + func TestLocation(t *testing.T) { suite.Run(t, new(locationTestSuite)) } From d360f20038b3623bb6becf295fcd7ae880b48431 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 19:01:53 +0530 Subject: [PATCH 16/22] Add Stat() test for os backend --- backend/os/file_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/backend/os/file_test.go b/backend/os/file_test.go index 46f4a44a..8d6b729d 100644 --- a/backend/os/file_test.go +++ b/backend/os/file_test.go @@ -719,6 +719,45 @@ func (s *osFileTest) TestLocationRightAfterChangeDir() { s.Contains(loc.Path(), "someDir/", "location now should contain 'someDir/'") } +// TestStat tests the Stat method returns correct file information +func (s *osFileTest) TestStat() { + // Create a temp file with content + tempFile, err := os.CreateTemp("", "os-stat-test") + s.NoError(err, "No error expected creating temp file") + defer func() { + tempFile.Close() + os.Remove(tempFile.Name()) + }() + + expectedContent := "test content for stat" + _, err = tempFile.Write([]byte(expectedContent)) + s.NoError(err, "No error expected writing to temp file") + tempFile.Close() // Close it so we can open through our API + + // Create VFS file from the temp file + fs := NewFileSystem() + file, err := fs.NewFile("", tempFile.Name()) + s.NoError(err, "No error expected creating VFS file") + + // Test Stat method + fileInfo, err := file.Stat() + s.NoError(err, "Stat() should not return an error for existing file") + s.NotNil(fileInfo, "FileInfo should not be nil") + + // Check file info properties + s.Equal(filepath.Base(tempFile.Name()), fileInfo.Name(), "FileInfo name should match file name") + s.Equal(int64(len(expectedContent)), fileInfo.Size(), "FileInfo size should match content length") + s.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") + s.NotNil(fileInfo.ModTime(), "ModTime should not be nil") + + // Test Stat on non-existent file + nonExistentFile, err := fs.NewFile("", "/non-existent-file.txt") + s.NoError(err, "error creating reference to non-existent file") + _, err = nonExistentFile.Stat() + s.Error(err, "error expected when calling Stat on non-existent file") + s.True(os.IsNotExist(err), "error should be os.IsNotExist") +} + func TestOSFile(t *testing.T) { suite.Run(t, new(osFileTest)) _ = os.Remove("test_files/new.txt") From ccde0cd50fd5df39b4fa3ed5c7b9fbcb58cd4c3f Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 19:03:33 +0530 Subject: [PATCH 17/22] Open test for os backend --- backend/os/location_test.go | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/backend/os/location_test.go b/backend/os/location_test.go index 1503be3b..d3f0dba9 100644 --- a/backend/os/location_test.go +++ b/backend/os/location_test.go @@ -1,6 +1,7 @@ package os import ( + "io/fs" "os" "path" "path/filepath" @@ -187,6 +188,61 @@ func (s *osLocationTest) TestDeleteFile() { s.False(exists, "Exists should return false after deleting the file.") } +// TestOpen tests the Open method in the Location implementation +func (s *osLocationTest) TestOpen() { + // Create a temp file with content + tempFileName := "open_test_file.txt" + testContent := "hello world test content" + + // Create file for testing within test directory structure + file, err := s.tmploc.NewFile("test_files/" + tempFileName) + s.NoError(err, "No error expected creating test file") + + _, err = file.Write([]byte(testContent)) + s.NoError(err, "Write should not error") + s.NoError(file.Close(), "Close should not error") + + // Get the file's location + loc := file.Location() + + // Test Opening the file + opened, err := loc.Open(tempFileName) + s.NoError(err, "Opening an existing file should not return an error") + s.NotNil(opened, "Opened file should not be nil") + + // Read the content to verify + data := make([]byte, len(testContent)) + n, err := opened.Read(data) + s.NoError(err, "Reading from opened file should not error") + s.Equal(len(testContent), n, "Should read all content") + s.Equal(testContent, string(data), "Content should match") + + // Test opening non-existent file + _, err = loc.Open("non-existent-file.txt") + s.Error(err, "Opening a non-existent file should return an error") + var pathErr *fs.PathError + s.ErrorAs(err, &pathErr, "Error should be a fs.PathError") + s.ErrorIs(pathErr.Err, fs.ErrNotExist, "Underlying error should be fs.ErrNotExist") + + // Test opening with path traversal attempts (should be rejected) + _, err = loc.Open("../outside.txt") + s.Error(err, "Opening a file with path traversal should return an error") + s.ErrorAs(err, &pathErr, "Error should be a fs.PathError") + s.ErrorIs(pathErr.Err, fs.ErrInvalid, "Underlying error should be fs.ErrInvalid") + + _, err = loc.Open("./file.txt") + s.Error(err, "Opening path with ./ should return an error") + + _, err = loc.Open(".") + s.Error(err, "Opening . should return an error") + + _, err = loc.Open("..") + s.Error(err, "Opening .. should return an error") + + // Clean up + s.NoError(file.Delete(), "Delete should not error") +} + func TestOSLocation(t *testing.T) { suite.Run(t, new(osLocationTest)) } From dcdae5cc2e9011e7f16e1ab6f86fea22ecaecb87 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 19:09:03 +0530 Subject: [PATCH 18/22] Add Stat() test for s3 backend --- backend/s3/file_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/backend/s3/file_test.go b/backend/s3/file_test.go index 7c54722b..07f5308d 100644 --- a/backend/s3/file_test.go +++ b/backend/s3/file_test.go @@ -970,6 +970,47 @@ func (ts *fileTestSuite) TestWriteOperations() { } } +// TestStat tests the functionality of the Stat method +func (ts *fileTestSuite) TestStat() { + // Setup expected values + contentLength := int64(42) + lastModified := time.Now() + + // Mock the S3 HeadObject response + headObjectOutput := &s3.HeadObjectOutput{ + ContentLength: aws.Int64(contentLength), + LastModified: aws.Time(lastModified), + } + s3cliMock.On("HeadObject", matchContext, mock.Anything).Return(headObjectOutput, nil).Once() + + // Call Stat + fileInfo, err := testFile.Stat() + + // Verify results + ts.NoError(err, "Stat() should not return an error for existing file") + ts.NotNil(fileInfo, "FileInfo should not be nil") + ts.Equal("file.txt", fileInfo.Name(), "FileInfo name should match file name") + ts.Equal(contentLength, fileInfo.Size(), "FileInfo size should match content length") + ts.Equal(lastModified.Truncate(time.Second), fileInfo.ModTime().Truncate(time.Second), "FileInfo modTime should match last modified time") + ts.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") + + // Verify mock expectations + s3cliMock.AssertExpectations(ts.T()) + + // Test Stat with S3 error + s3Error := errors.New("s3 error") + s3cliMock.On("HeadObject", matchContext, mock.Anything).Return(nil, s3Error).Once() + + // Call Stat again, expecting an error + _, err = testFile.Stat() + ts.Error(err, "Stat() should return an error when S3 HeadObject fails") + ts.True(errors.Is(err, s3Error) || strings.Contains(err.Error(), s3Error.Error()), + "error should contain the original S3 error") + + // Verify mock expectations + s3cliMock.AssertExpectations(ts.T()) +} + func TestFile(t *testing.T) { suite.Run(t, new(fileTestSuite)) } From 42faa1d93c250257df6579e2787d47207893b966 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 19:11:45 +0530 Subject: [PATCH 19/22] Add Open() test for s3 backend --- backend/gs/location_test.go | 4 +-- backend/mem/file_test.go | 4 +-- backend/mem/location_test.go | 18 ++++++------ backend/os/file_test.go | 2 +- backend/s3/location_test.go | 56 ++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/backend/gs/location_test.go b/backend/gs/location_test.go index c8b7d70c..0740a791 100644 --- a/backend/gs/location_test.go +++ b/backend/gs/location_test.go @@ -365,12 +365,12 @@ func (lt *locationTestSuite) TestOpen() { // Test opening a non-existent file _, err = loc.Open("non-existent-file.txt") lt.Error(err, "Opening a non-existent file should return an error") - lt.Contains(err.Error(), "ErrNotExist", "Error should indicate file does not exist") + lt.Contains(err.Error(), "file does not exist", "Error should indicate file does not exist") // Test opening with path traversal _, err = loc.Open("../outside.txt") lt.Error(err, "Opening a file with path traversal should return an error") - lt.Contains(err.Error(), "ErrInvalid", "Error should indicate invalid path") + lt.Contains(err.Error(), "invalid argument", "Error should indicate invalid path") // Test opening with dot paths _, err = loc.Open(".") diff --git a/backend/mem/file_test.go b/backend/mem/file_test.go index 84ecec34..1562d582 100644 --- a/backend/mem/file_test.go +++ b/backend/mem/file_test.go @@ -770,14 +770,14 @@ func (s *memFileTest) TestStat() { fileInfo, err := s.testFile.Stat() s.NoError(err, "stat error not expected") s.NotNil(fileInfo, "FileInfo should not be nil") - + // Check file info properties s.Equal("test.txt", fileInfo.Name(), "FileInfo name should match file name") s.Equal(int64(len(expectedContent)), fileInfo.Size(), "FileInfo size should match content length") s.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") s.NotNil(fileInfo.ModTime(), "ModTime should not be nil") s.Equal(0644, int(fileInfo.Mode()), "Mode should be 0644") - + // Test Stat on non-existent file nonExistentFile, err := s.fileSystem.NewFile("", "/non-existent-file.txt") s.NoError(err, "error creating reference to non-existent file") diff --git a/backend/mem/location_test.go b/backend/mem/location_test.go index a10458fb..47d8e028 100644 --- a/backend/mem/location_test.go +++ b/backend/mem/location_test.go @@ -350,48 +350,48 @@ func (s *memLocationTest) TestOpen() { testContent := "hello world" testFileName := "open_test_file.txt" testFilePath := "/test_files/" + testFileName - + // Create a file and write content file, err := s.fileSystem.NewFile("", testFilePath) s.NoError(err, "error not expected when creating a new file") _, err = file.Write([]byte(testContent)) s.NoError(err, "write error not expected") s.NoError(file.Close(), "close error not expected") - + // Get the file's location loc := file.Location() - + // Test Opening the file opened, err := loc.Open(testFileName) s.NoError(err, "error not expected when opening existing file") s.NotNil(opened, "opened file should not be nil") - + // Read the content to verify data := make([]byte, len(testContent)) n, err := opened.Read(data) s.NoError(err, "read error not expected") s.Equal(len(testContent), n, "should read all content") s.Equal(testContent, string(data), "content should match") - + // Test opening non-existent file _, err = loc.Open("non-existent-file.txt") s.Error(err, "error expected when opening non-existent file") var pathErr *fs.PathError s.True(errors.As(err, &pathErr), "error should be a fs.PathError") s.ErrorIs(pathErr.Err, fs.ErrNotExist, "underlying error should be fs.ErrNotExist") - + // Test opening with path traversal attempts (should be rejected) _, err = loc.Open("../outside.txt") s.Error(err, "error expected when path contains traversal") s.True(errors.As(err, &pathErr), "error should be a fs.PathError") s.ErrorIs(pathErr.Err, fs.ErrInvalid, "underlying error should be fs.ErrInvalid") - + _, err = loc.Open("./file.txt") s.Error(err, "error expected when path contains ./") - + _, err = loc.Open(".") s.Error(err, "error expected when path is .") - + _, err = loc.Open("..") s.Error(err, "error expected when path is ..") } diff --git a/backend/os/file_test.go b/backend/os/file_test.go index 8d6b729d..1befcf00 100644 --- a/backend/os/file_test.go +++ b/backend/os/file_test.go @@ -755,7 +755,7 @@ func (s *osFileTest) TestStat() { s.NoError(err, "error creating reference to non-existent file") _, err = nonExistentFile.Stat() s.Error(err, "error expected when calling Stat on non-existent file") - s.True(os.IsNotExist(err), "error should be os.IsNotExist") + s.Contains(err.Error(), "no such file or directory", "error should indicate no such file") } func TestOSFile(t *testing.T) { diff --git a/backend/s3/location_test.go b/backend/s3/location_test.go index 3eefc672..c31a4e4a 100644 --- a/backend/s3/location_test.go +++ b/backend/s3/location_test.go @@ -4,6 +4,7 @@ import ( "path" "regexp" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -342,6 +343,61 @@ func (lt *locationTestSuite) TestDeleteFileWithAllVersionsOption() { lt.s3cliMock.AssertNumberOfCalls(lt.T(), "DeleteObject", 3) } +// TestOpen tests the Open method for the S3 location +func (lt *locationTestSuite) TestOpen() { + // Setup test values + bucket := "test-bucket" + locPath := "/test-dir/" + filename := "test-file.txt" + + // Create location + loc, err := lt.fs.NewLocation(bucket, locPath) + lt.NoError(err, "Creating location shouldn't return an error") + + // Mock the HeadObject response for checking existence + lt.s3cliMock.On("HeadObject", matchContext, mock.Anything).Return(&s3.HeadObjectOutput{ + ContentLength: aws.Int64(42), + LastModified: aws.Time(time.Now()), + }, nil).Once() + + // Test opening an existing file + file, err := loc.Open(filename) + lt.NoError(err, "Opening an existing file should not return an error") + lt.NotNil(file, "Returned file should not be nil") + + // Verify mock expectations + lt.s3cliMock.AssertExpectations(lt.T()) + + // Test opening a non-existent file + // Setup mock to indicate file doesn't exist + lt.s3cliMock.On("HeadObject", matchContext, mock.Anything).Return(nil, &types.NotFound{ + Message: aws.String("Not Found"), + }).Once() + + // Attempt to open a non-existent file + _, err = loc.Open("non-existent.txt") + lt.Error(err, "Opening a non-existent file should return an error") + lt.Contains(err.Error(), "file does not exist", "Error should indicate file does not exist") + + // Test opening with path traversal + _, err = loc.Open("../outside.txt") + lt.Error(err, "Opening a file with path traversal should return an error") + lt.Contains(err.Error(), "invalid argument", "Error should indicate invalid path") + + // Test opening with dot paths + _, err = loc.Open(".") + lt.Error(err, "Opening '.' should return an error") + + _, err = loc.Open("..") + lt.Error(err, "Opening '..' should return an error") + + _, err = loc.Open("./file.txt") + lt.Error(err, "Opening path with './' should return an error") + + // Verify all mock expectations + lt.s3cliMock.AssertExpectations(lt.T()) +} + func TestLocation(t *testing.T) { suite.Run(t, new(locationTestSuite)) } From 31693b7520827946a75b16111ca86d283af8c69e Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 20:28:31 +0530 Subject: [PATCH 20/22] Add new test to azure file test --- backend/azure/file_test.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/backend/azure/file_test.go b/backend/azure/file_test.go index bd30fdf4..882ff802 100644 --- a/backend/azure/file_test.go +++ b/backend/azure/file_test.go @@ -410,6 +410,43 @@ func (s *FileTestSuite) TestIsSameAuth_DifferentAcctKey() { s.False(sourceFile.isSameAuth(targetFile), "Files were created with different account keys so same auth should be false") } +func (s *FileTestSuite) TestStat() { + // Setup test values + fileContent := "Hello, this is a test file content." + modTime := time.Now().UTC() + + // Mock properties response + client := MockAzureClient{ + PropertiesResult: &BlobProperties{ + Size: to.Ptr(int64(len(fileContent))), + LastModified: to.Ptr(modTime), + }, + } + fs := NewFileSystem(WithClient(&client)) + + // Create a test file + f, err := fs.NewFile("test-container", "/foo/bar.txt") + s.NoError(err, "Creating file shouldn't return an error") + + // Test successful stat + fileInfo, err := f.Stat() + s.NoError(err, "Stat() should not return an error for existing file") + s.NotNil(fileInfo, "FileInfo should not be nil") + s.Equal("bar.txt", fileInfo.Name(), "FileInfo name should match file name") + s.Equal(int64(len(fileContent)), fileInfo.Size(), "FileInfo size should match content length") + s.False(fileInfo.IsDir(), "FileInfo should indicate file is not a directory") + s.Equal(modTime, fileInfo.ModTime(), "ModTime should match") + + // Test error case + client = MockAzureClient{PropertiesError: errors.New("blob not found")} + fs = NewFileSystem(WithClient(&client)) + f, _ = fs.NewFile("test-container", "/foo/non-existent.txt") + + _, err = f.Stat() + s.Error(err, "Stat() should return an error for non-existent file") + s.Contains(err.Error(), "blob not found", "Error should indicate file does not exist") +} + func TestAzureFile(t *testing.T) { suite.Run(t, new(FileTestSuite)) } From 10e30807cd68f2287d647d25926159a6ba011fc9 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 20:36:47 +0530 Subject: [PATCH 21/22] Fix windows os backend test issue --- backend/os/file_test.go | 5 ++++- backend/os/location_test.go | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/os/file_test.go b/backend/os/file_test.go index 1befcf00..d498da5d 100644 --- a/backend/os/file_test.go +++ b/backend/os/file_test.go @@ -7,6 +7,7 @@ import ( "os" "path" "path/filepath" + "strings" "testing" "time" @@ -755,7 +756,9 @@ func (s *osFileTest) TestStat() { s.NoError(err, "error creating reference to non-existent file") _, err = nonExistentFile.Stat() s.Error(err, "error expected when calling Stat on non-existent file") - s.Contains(err.Error(), "no such file or directory", "error should indicate no such file") + // Use Contains with platform-independent error messages + errorMsg := err.Error() + s.True(strings.Contains(errorMsg, "no such file") || strings.Contains(errorMsg, "cannot find the file"), "error should indicate no such file") } func TestOSFile(t *testing.T) { diff --git a/backend/os/location_test.go b/backend/os/location_test.go index d3f0dba9..94c522b8 100644 --- a/backend/os/location_test.go +++ b/backend/os/location_test.go @@ -216,6 +216,9 @@ func (s *osLocationTest) TestOpen() { s.NoError(err, "Reading from opened file should not error") s.Equal(len(testContent), n, "Should read all content") s.Equal(testContent, string(data), "Content should match") + + // Close the opened file to avoid "file in use" errors on Windows + s.NoError(opened.Close(), "Closing opened file should not error") // Test opening non-existent file _, err = loc.Open("non-existent-file.txt") From 8349113da4f54ad64b09eeabd8f29734ebe0abb2 Mon Sep 17 00:00:00 2001 From: ThisaraWeerakoon Date: Sun, 11 May 2025 20:44:44 +0530 Subject: [PATCH 22/22] Fix lint issues --- backend/os/file_test.go | 13 ++++++++----- backend/os/location_test.go | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/os/file_test.go b/backend/os/file_test.go index d498da5d..881f9c1f 100644 --- a/backend/os/file_test.go +++ b/backend/os/file_test.go @@ -726,14 +726,15 @@ func (s *osFileTest) TestStat() { tempFile, err := os.CreateTemp("", "os-stat-test") s.NoError(err, "No error expected creating temp file") defer func() { - tempFile.Close() - os.Remove(tempFile.Name()) + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) }() expectedContent := "test content for stat" - _, err = tempFile.Write([]byte(expectedContent)) + _, err = tempFile.WriteString(expectedContent) s.NoError(err, "No error expected writing to temp file") - tempFile.Close() // Close it so we can open through our API + err = tempFile.Close() // Close it so we can open through our API + s.NoError(err, "No error expected when closing temp file") // Create VFS file from the temp file fs := NewFileSystem() @@ -758,7 +759,9 @@ func (s *osFileTest) TestStat() { s.Error(err, "error expected when calling Stat on non-existent file") // Use Contains with platform-independent error messages errorMsg := err.Error() - s.True(strings.Contains(errorMsg, "no such file") || strings.Contains(errorMsg, "cannot find the file"), "error should indicate no such file") + hasNoSuchFile := strings.Contains(errorMsg, "no such file") + cannotFindFile := strings.Contains(errorMsg, "cannot find the file") + s.True(hasNoSuchFile || cannotFindFile, "error should indicate no such file") } func TestOSFile(t *testing.T) { diff --git a/backend/os/location_test.go b/backend/os/location_test.go index 94c522b8..2def32c8 100644 --- a/backend/os/location_test.go +++ b/backend/os/location_test.go @@ -216,7 +216,7 @@ func (s *osLocationTest) TestOpen() { s.NoError(err, "Reading from opened file should not error") s.Equal(len(testContent), n, "Should read all content") s.Equal(testContent, string(data), "Content should match") - + // Close the opened file to avoid "file in use" errors on Windows s.NoError(opened.Close(), "Closing opened file should not error")