From 7b7e0bb56dd2d37fb4e387cb2f60c8ac6e99ac9f Mon Sep 17 00:00:00 2001 From: Erik Unger Date: Thu, 9 Nov 2023 23:33:43 +0100 Subject: [PATCH] ListDirMaxFileSystem and ListDirRecursiveFileSystem --- dropboxfs/dropboxfs.go | 6 --- file.go | 98 +++++++++++++++++++++---------------- file_test.go | 24 +++++++++ filesystem.go | 30 ++++++------ other.go => fsimpl/other.go | 2 +- listdir.go | 33 ++----------- listdir_test.go | 4 +- localfilesystem.go | 9 ---- multipartfs/multipartfs.go | 10 ---- s3fs/s3fs.go | 6 --- utils.go | 4 +- zipfs/zipfs.go | 9 ---- 12 files changed, 106 insertions(+), 129 deletions(-) rename other.go => fsimpl/other.go (98%) diff --git a/dropboxfs/dropboxfs.go b/dropboxfs/dropboxfs.go index 71f2e59..246ab9e 100644 --- a/dropboxfs/dropboxfs.go +++ b/dropboxfs/dropboxfs.go @@ -290,12 +290,6 @@ func (dbfs *DropboxFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath return dbfs.listDirInfo(ctx, dirPath, callback, patterns, true) } -func (dbfs *DropboxFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) { - return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error { - return dbfs.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns) - }) -} - func (dbfs *DropboxFileSystem) Touch(filePath string, perm []fs.Permissions) error { if dbfs.info(filePath).Exists { return errors.New("Touch can't change time on Dropbox") diff --git a/file.go b/file.go index 3a77457..d9be4cd 100644 --- a/file.go +++ b/file.go @@ -401,11 +401,7 @@ func (file File) SetPermissions(perm Permissions) error { // If any patterns are passed, then only files with a name that matches // at least one of the patterns are returned. func (file File) ListDir(callback func(File) error, patterns ...string) error { - if file == "" { - return ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfo(context.Background(), path, FileInfoCallback(callback), patterns) + return file.ListDirContext(context.Background(), callback, patterns...) } // ListDirContext calls the passed callback function for every file and directory in dirPath. @@ -416,18 +412,14 @@ func (file File) ListDirContext(ctx context.Context, callback func(File) error, return ErrEmptyPath } fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfo(ctx, path, FileInfoCallback(callback), patterns) + return fileSystem.ListDirInfo(ctx, path, FileInfoToFileCallback(callback), patterns) } // ListDirInfo calls the passed callback function for every file and directory in dirPath. // If any patterns are passed, then only files with a name that matches // at least one of the patterns are returned. func (file File) ListDirInfo(callback func(*FileInfo) error, patterns ...string) error { - if file == "" { - return ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfo(context.Background(), path, callback, patterns) + return file.ListDirInfoContext(context.Background(), callback, patterns...) } // ListDirInfoContext calls the passed callback function for every file and directory in dirPath. @@ -444,21 +436,13 @@ func (file File) ListDirInfoContext(ctx context.Context, callback func(*FileInfo // ListDirRecursive returns only files. // patterns are only applied to files, not to directories func (file File) ListDirRecursive(callback func(File) error, patterns ...string) error { - if file == "" { - return ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfoRecursive(context.Background(), path, FileInfoCallback(callback), patterns) + return file.ListDirInfoRecursiveContext(context.Background(), FileInfoToFileCallback(callback), patterns...) } // ListDirRecursiveContext returns only files. // patterns are only applied to files, not to directories func (file File) ListDirRecursiveContext(ctx context.Context, callback func(File) error, patterns ...string) error { - if file == "" { - return ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfoRecursive(ctx, path, FileInfoCallback(callback), patterns) + return file.ListDirInfoRecursiveContext(ctx, FileInfoToFileCallback(callback), patterns...) } // ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath @@ -466,15 +450,11 @@ func (file File) ListDirRecursiveContext(ctx context.Context, callback func(File // If any patterns are passed, then only files (not directories) with a name that matches // at least one of the patterns are returned. func (file File) ListDirInfoRecursive(callback func(*FileInfo) error, patterns ...string) error { - if file == "" { - return ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfoRecursive(context.Background(), path, callback, patterns) + return file.ListDirInfoRecursiveContext(context.Background(), callback, patterns...) } -// ListDirInfoRecursiveContext calls the passed callback function for every file (not directory) in dirPath -// recursing into all sub-directories. +// ListDirInfoRecursiveContext calls the passed callback function for every file +// (not directory) in dirPath recursing into all sub-directories. // If any patterns are passed, then only files (not directories) with a name that matches // at least one of the patterns are returned. func (file File) ListDirInfoRecursiveContext(ctx context.Context, callback func(*FileInfo) error, patterns ...string) error { @@ -482,7 +462,25 @@ func (file File) ListDirInfoRecursiveContext(ctx context.Context, callback func( return ErrEmptyPath } fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirInfoRecursive(ctx, path, callback, patterns) + if fs, ok := fileSystem.(ListDirRecursiveFileSystem); ok { + return fs.ListDirInfoRecursive(ctx, path, callback, patterns) + } + return fileSystem.ListDirInfo(ctx, path, + func(info *FileInfo) error { + if info.IsDir { + // Not returning directories, but recursing into them + err := info.File.ListDirInfoRecursiveContext(ctx, callback, patterns...) + // Don't mind files that have been deleted while iterating + return RemoveErrDoesNotExist(err) + } + match, err := fileSystem.MatchAnyPattern(info.Name, patterns) + if !match || err != nil { + return err + } + return callback(info) + }, + nil, // No patterns + ) } // ListDirMax returns at most max files and directories in dirPath. @@ -490,11 +488,7 @@ func (file File) ListDirInfoRecursiveContext(ctx context.Context, callback func( // If any patterns are passed, then only files or directories with a name that matches // at least one of the patterns are returned. func (file File) ListDirMax(max int, patterns ...string) (files []File, err error) { - if file == "" { - return nil, ErrEmptyPath - } - fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirMax(context.Background(), path, max, patterns) + return file.ListDirMaxContext(context.Background(), max, patterns...) } // ListDirMaxContext returns at most max files and directories in dirPath. @@ -505,24 +499,44 @@ func (file File) ListDirMaxContext(ctx context.Context, max int, patterns ...str if file == "" { return nil, ErrEmptyPath } + if max == 0 { + return nil, nil + } fileSystem, path := file.ParseRawURI() - return fileSystem.ListDirMax(ctx, path, max, patterns) + if fs, ok := fileSystem.(ListDirMaxFileSystem); ok { + return fs.ListDirMax(ctx, path, max, patterns) + } + done := errors.New("done") // used as an internal flag, won't be returned + err = fileSystem.ListDirInfo(ctx, path, func(info *FileInfo) error { + if max >= 0 && len(files) >= max { + return done + } + if files == nil { + // Reserve space for files + capacity := max + if capacity < 0 { + capacity = 64 + } + files = make([]File, 0, capacity) + } + files = append(files, info.File) + return nil + }, patterns) + if err != nil && !errors.Is(err, done) { + return nil, err + } + return files, nil } func (file File) ListDirRecursiveMax(max int, patterns ...string) (files []File, err error) { - if file == "" { - return nil, ErrEmptyPath - } - return ListDirMaxImpl(context.Background(), max, func(ctx context.Context, callback func(File) error) error { - return file.ListDirRecursiveContext(ctx, callback, patterns...) - }) + return file.ListDirRecursiveMaxContext(context.Background(), max, patterns...) } func (file File) ListDirRecursiveMaxContext(ctx context.Context, max int, patterns ...string) (files []File, err error) { if file == "" { return nil, ErrEmptyPath } - return ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(File) error) error { + return listDirMaxImpl(ctx, max, func(ctx context.Context, callback func(File) error) error { return file.ListDirRecursiveContext(ctx, callback, patterns...) }) } diff --git a/file_test.go b/file_test.go index 2276d00..efa8385 100644 --- a/file_test.go +++ b/file_test.go @@ -1,6 +1,7 @@ package fs import ( + "context" "errors" "fmt" "testing" @@ -200,3 +201,26 @@ func TestFile_Watch(t *testing.T) { err = cancel() assert.NoError(t, err, "cancel watch") } + +func TestFile_ListDirInfoRecursiveContext(t *testing.T) { + type args struct { + ctx context.Context + callback func(*FileInfo) error + patterns []string + } + tests := []struct { + name string + file File + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.file.ListDirInfoRecursiveContext(tt.args.ctx, tt.args.callback, tt.args.patterns...); (err != nil) != tt.wantErr { + t.Errorf("File.ListDirInfoRecursiveContext() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/filesystem.go b/filesystem.go index 63f13a2..63a131c 100644 --- a/filesystem.go +++ b/filesystem.go @@ -79,20 +79,6 @@ type FileSystem interface { // at least one of the patterns are returned. ListDirInfo(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error - // TODO - // ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath - // recursing into all sub-directories. - // If any patterns are passed, then only files (not directories) with a name that matches - // at least one of the patterns are returned. - ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error - - // TODO - // ListDirMax returns at most max files and directories in dirPath. - // A max value of -1 returns all files. - // If any patterns are passed, then only files or directories with a name that matches - // at least one of the patterns are returned. - ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) ([]File, error) - MakeDir(dirPath string, perm []Permissions) error OpenReader(filePath string) (ReadCloser, error) @@ -271,3 +257,19 @@ type MakeAllDirsFileSystem interface { MakeAllDirs(dirPath string, perm []Permissions) error } + +type ListDirMaxFileSystem interface { + // ListDirMax returns at most max files and directories in dirPath. + // A max value of -1 returns all files. + // If any patterns are passed, then only files or directories with a name that matches + // at least one of the patterns are returned. + ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) ([]File, error) +} + +type ListDirRecursiveFileSystem interface { + // ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath + // recursing into all sub-directories. + // If any patterns are passed, then only files (not directories) with a name that matches + // at least one of the patterns are returned. + ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error +} diff --git a/other.go b/fsimpl/other.go similarity index 98% rename from other.go rename to fsimpl/other.go index e506895..4c15a64 100644 --- a/other.go +++ b/fsimpl/other.go @@ -1,4 +1,4 @@ -package fs +package fsimpl /* import ( diff --git a/listdir.go b/listdir.go index 366c039..4dfa4a5 100644 --- a/listdir.go +++ b/listdir.go @@ -5,21 +5,21 @@ import ( "errors" ) -// ListDirMaxImpl implements the ListDirMax method functionality by calling listDir. +// listDirMaxImpl implements the ListDirMax method functionality by calling listDir. // It returns the passed max number of files or an unlimited number if max is < 0. // FileSystem implementations can use this function to implement ListDirMax, // if an own, specialized implementation doesn't make sense. -func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Context, callback func(File) error) error) (files []File, err error) { +func listDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Context, callback func(File) error) error) (files []File, err error) { if ctx.Err() != nil { return nil, ctx.Err() } if max == 0 { return nil, nil } - errAbort := errors.New("errAbort") // used as an internal flag, won't be returned + done := errors.New("done") // used as an internal flag, won't be returned err = listDir(ctx, func(file File) error { if max >= 0 && len(files) >= max { - return errAbort + return done } if files == nil { // Reserve space for files @@ -32,7 +32,7 @@ func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Conte files = append(files, file) return nil }) - if err != nil && !errors.Is(err, errAbort) { + if err != nil && !errors.Is(err, done) { return nil, err } return files, nil @@ -55,26 +55,3 @@ func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Conte // return err // }, nil) // } - -// ListDirInfoRecursiveImpl can be used by FileSystem implementations to -// implement FileSystem.ListDirRecursive if it doesn't have an internal -// optimzed implementation for it. -func ListDirInfoRecursiveImpl(ctx context.Context, fs FileSystem, dirPath string, callback func(*FileInfo) error, patterns []string) error { - return fs.ListDirInfo( - ctx, - dirPath, - func(info *FileInfo) error { - if info.IsDir { - err := info.File.ListDirInfoRecursiveContext(ctx, callback, patterns...) - // Don't mind files that have been deleted while iterating - return RemoveErrDoesNotExist(err) - } - match, err := fs.MatchAnyPattern(info.Name, patterns) - if !match || err != nil { - return err - } - return callback(info) - }, - nil, // No patterns - ) -} diff --git a/listdir_test.go b/listdir_test.go index 695d053..4818d1e 100644 --- a/listdir_test.go +++ b/listdir_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func TestListDirMaxImpl(t *testing.T) { +func Test_listDirMaxImpl(t *testing.T) { bg := context.Background() errCtx, cancel := context.WithCancel(context.Background()) cancel() @@ -47,7 +47,7 @@ func TestListDirMaxImpl(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotFiles, err := ListDirMaxImpl(tt.args.ctx, tt.args.max, tt.args.listDir) + gotFiles, err := listDirMaxImpl(tt.args.ctx, tt.args.max, tt.args.listDir) if (err != nil) != tt.wantErr { t.Errorf("ListDirMaxImpl() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/localfilesystem.go b/localfilesystem.go index 667347c..57c68d4 100644 --- a/localfilesystem.go +++ b/localfilesystem.go @@ -321,15 +321,6 @@ func (local *LocalFileSystem) ListDirInfo(ctx context.Context, dirPath string, c return nil } -func (local *LocalFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error { - if dirPath == "" { - return ErrEmptyPath - } - dirPath = filepath.Clean(dirPath) - dirPath = expandTilde(dirPath) - return ListDirInfoRecursiveImpl(ctx, local, dirPath, callback, patterns) -} - func (local *LocalFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []File, err error) { if ctx.Err() != nil { return nil, ctx.Err() diff --git a/multipartfs/multipartfs.go b/multipartfs/multipartfs.go index ddafae3..ae9ea9b 100644 --- a/multipartfs/multipartfs.go +++ b/multipartfs/multipartfs.go @@ -254,16 +254,6 @@ func (mpfs *MultipartFileSystem) ListDirInfo(ctx context.Context, dirPath string return err } -func (mpfs *MultipartFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error { - return fs.ListDirInfoRecursiveImpl(ctx, mpfs, dirPath, callback, patterns) -} - -func (mpfs *MultipartFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) { - return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error { - return mpfs.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns) - }) -} - func (mpfs *MultipartFileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) { if ctx.Err() != nil { return nil, ctx.Err() diff --git a/s3fs/s3fs.go b/s3fs/s3fs.go index 2b3b9e2..8a29c99 100644 --- a/s3fs/s3fs.go +++ b/s3fs/s3fs.go @@ -281,12 +281,6 @@ func (s *S3FileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, return s.listDirInfo(ctx, dirPath, callback, patterns, true) } -func (s *S3FileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) { - return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error { - return s.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns) - }) -} - func (s *S3FileSystem) Touch(filePath string, perm []fs.Permissions) error { if s.Exists(filePath) { return nil // TODO is this OK, can we change the modified time? diff --git a/utils.go b/utils.go index cf8e7bb..eb5b689 100644 --- a/utils.go +++ b/utils.go @@ -59,10 +59,10 @@ func StringsToFileReaders(fileURIs []string) []FileReader { return fileReaders } -// FileInfoCallback converts a File callback function +// FileInfoToFileCallback converts a File callback function // into a FileInfo callback function that is calling // the passed fileCallback with the FileInfo.File. -func FileInfoCallback(fileCallback func(File) error) func(*FileInfo) error { +func FileInfoToFileCallback(fileCallback func(File) error) func(*FileInfo) error { return func(info *FileInfo) error { return fileCallback(info.File) } diff --git a/zipfs/zipfs.go b/zipfs/zipfs.go index 035bfbb..3f87fae 100644 --- a/zipfs/zipfs.go +++ b/zipfs/zipfs.go @@ -305,15 +305,6 @@ func (zipfs *ZipFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath st return listRecursive(rootNode) } -func (zipfs *ZipFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) { - if zipfs.zipReader == nil { - return nil, fs.ErrWriteOnlyFileSystem - } - return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error { - return zipfs.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns) - }) -} - func (zipfs *ZipFileSystem) OpenReader(filePath string) (iofs.File, error) { if zipfs.zipReader == nil { return nil, fs.ErrWriteOnlyFileSystem