ListDirMaxFileSystem and ListDirRecursiveFileSystem

dropboxfs/dropboxfs.go

@@ -290,12 +290,6 @@ func (dbfs *DropboxFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath
 	return dbfs.listDirInfo(ctx, dirPath, callback, patterns, true)
 }
 
-func (dbfs *DropboxFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) {
-	return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error {
-		return dbfs.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns)
-	})
-}
-
 func (dbfs *DropboxFileSystem) Touch(filePath string, perm []fs.Permissions) error {
 	if dbfs.info(filePath).Exists {
 		return errors.New("Touch can't change time on Dropbox")

file.go

@@ -401,11 +401,7 @@ func (file File) SetPermissions(perm Permissions) error {
 // If any patterns are passed, then only files with a name that matches
 // at least one of the patterns are returned.
 func (file File) ListDir(callback func(File) error, patterns ...string) error {
-	if file == "" {
-		return ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfo(context.Background(), path, FileInfoCallback(callback), patterns)
+	return file.ListDirContext(context.Background(), callback, patterns...)
 }
 
 // ListDirContext calls the passed callback function for every file and directory in dirPath.
@@ -416,18 +412,14 @@ func (file File) ListDirContext(ctx context.Context, callback func(File) error,
 		return ErrEmptyPath
 	}
 	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfo(ctx, path, FileInfoCallback(callback), patterns)
+	return fileSystem.ListDirInfo(ctx, path, FileInfoToFileCallback(callback), patterns)
 }
 
 // ListDirInfo calls the passed callback function for every file and directory in dirPath.
 // If any patterns are passed, then only files with a name that matches
 // at least one of the patterns are returned.
 func (file File) ListDirInfo(callback func(*FileInfo) error, patterns ...string) error {
-	if file == "" {
-		return ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfo(context.Background(), path, callback, patterns)
+	return file.ListDirInfoContext(context.Background(), callback, patterns...)
 }
 
 // ListDirInfoContext calls the passed callback function for every file and directory in dirPath.
@@ -444,57 +436,59 @@ func (file File) ListDirInfoContext(ctx context.Context, callback func(*FileInfo
 // ListDirRecursive returns only files.
 // patterns are only applied to files, not to directories
 func (file File) ListDirRecursive(callback func(File) error, patterns ...string) error {
-	if file == "" {
-		return ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfoRecursive(context.Background(), path, FileInfoCallback(callback), patterns)
+	return file.ListDirInfoRecursiveContext(context.Background(), FileInfoToFileCallback(callback), patterns...)
 }
 
 // ListDirRecursiveContext returns only files.
 // patterns are only applied to files, not to directories
 func (file File) ListDirRecursiveContext(ctx context.Context, callback func(File) error, patterns ...string) error {
-	if file == "" {
-		return ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfoRecursive(ctx, path, FileInfoCallback(callback), patterns)
+	return file.ListDirInfoRecursiveContext(ctx, FileInfoToFileCallback(callback), patterns...)
 }
 
 // ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath
 // recursing into all sub-directories.
 // If any patterns are passed, then only files (not directories) with a name that matches
 // at least one of the patterns are returned.
 func (file File) ListDirInfoRecursive(callback func(*FileInfo) error, patterns ...string) error {
-	if file == "" {
-		return ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfoRecursive(context.Background(), path, callback, patterns)
+	return file.ListDirInfoRecursiveContext(context.Background(), callback, patterns...)
 }
 
-// ListDirInfoRecursiveContext calls the passed callback function for every file (not directory) in dirPath
-// recursing into all sub-directories.
+// ListDirInfoRecursiveContext calls the passed callback function for every file
+// (not directory) in dirPath recursing into all sub-directories.
 // If any patterns are passed, then only files (not directories) with a name that matches
 // at least one of the patterns are returned.
 func (file File) ListDirInfoRecursiveContext(ctx context.Context, callback func(*FileInfo) error, patterns ...string) error {
 	if file == "" {
 		return ErrEmptyPath
 	}
 	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirInfoRecursive(ctx, path, callback, patterns)
+	if fs, ok := fileSystem.(ListDirRecursiveFileSystem); ok {
+		return fs.ListDirInfoRecursive(ctx, path, callback, patterns)
+	}
+	return fileSystem.ListDirInfo(ctx, path,
+		func(info *FileInfo) error {
+			if info.IsDir {
+				// Not returning directories, but recursing into them
+				err := info.File.ListDirInfoRecursiveContext(ctx, callback, patterns...)
+				// Don't mind files that have been deleted while iterating
+				return RemoveErrDoesNotExist(err)
+			}
+			match, err := fileSystem.MatchAnyPattern(info.Name, patterns)
+			if !match || err != nil {
+				return err
+			}
+			return callback(info)
+		},
+		nil, // No patterns
+	)
 }
 
 // ListDirMax returns at most max files and directories in dirPath.
 // A max value of -1 returns all files.
 // If any patterns are passed, then only files or directories with a name that matches
 // at least one of the patterns are returned.
 func (file File) ListDirMax(max int, patterns ...string) (files []File, err error) {
-	if file == "" {
-		return nil, ErrEmptyPath
-	}
-	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirMax(context.Background(), path, max, patterns)
+	return file.ListDirMaxContext(context.Background(), max, patterns...)
 }
 
 // ListDirMaxContext returns at most max files and directories in dirPath.
@@ -505,24 +499,44 @@ func (file File) ListDirMaxContext(ctx context.Context, max int, patterns ...str
 	if file == "" {
 		return nil, ErrEmptyPath
 	}
+	if max == 0 {
+		return nil, nil
+	}
 	fileSystem, path := file.ParseRawURI()
-	return fileSystem.ListDirMax(ctx, path, max, patterns)
+	if fs, ok := fileSystem.(ListDirMaxFileSystem); ok {
+		return fs.ListDirMax(ctx, path, max, patterns)
+	}
+	done := errors.New("done") // used as an internal flag, won't be returned
+	err = fileSystem.ListDirInfo(ctx, path, func(info *FileInfo) error {
+		if max >= 0 && len(files) >= max {
+			return done
+		}
+		if files == nil {
+			// Reserve space for files
+			capacity := max
+			if capacity < 0 {
+				capacity = 64
+			}
+			files = make([]File, 0, capacity)
+		}
+		files = append(files, info.File)
+		return nil
+	}, patterns)
+	if err != nil && !errors.Is(err, done) {
+		return nil, err
+	}
+	return files, nil
 }
 
 func (file File) ListDirRecursiveMax(max int, patterns ...string) (files []File, err error) {
-	if file == "" {
-		return nil, ErrEmptyPath
-	}
-	return ListDirMaxImpl(context.Background(), max, func(ctx context.Context, callback func(File) error) error {
-		return file.ListDirRecursiveContext(ctx, callback, patterns...)
-	})
+	return file.ListDirRecursiveMaxContext(context.Background(), max, patterns...)
 }
 
 func (file File) ListDirRecursiveMaxContext(ctx context.Context, max int, patterns ...string) (files []File, err error) {
 	if file == "" {
 		return nil, ErrEmptyPath
 	}
-	return ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(File) error) error {
+	return listDirMaxImpl(ctx, max, func(ctx context.Context, callback func(File) error) error {
 		return file.ListDirRecursiveContext(ctx, callback, patterns...)
 	})
 }

file_test.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"testing"
@@ -200,3 +201,26 @@ func TestFile_Watch(t *testing.T) {
 	err = cancel()
 	assert.NoError(t, err, "cancel watch")
 }
+
+func TestFile_ListDirInfoRecursiveContext(t *testing.T) {
+	type args struct {
+		ctx      context.Context
+		callback func(*FileInfo) error
+		patterns []string
+	}
+	tests := []struct {
+		name    string
+		file    File
+		args    args
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.file.ListDirInfoRecursiveContext(tt.args.ctx, tt.args.callback, tt.args.patterns...); (err != nil) != tt.wantErr {
+				t.Errorf("File.ListDirInfoRecursiveContext() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

filesystem.go

@@ -79,20 +79,6 @@ type FileSystem interface {
 	// at least one of the patterns are returned.
 	ListDirInfo(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error
 
-	// TODO
-	// ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath
-	// recursing into all sub-directories.
-	// If any patterns are passed, then only files (not directories) with a name that matches
-	// at least one of the patterns are returned.
-	ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error
-
-	// TODO
-	// ListDirMax returns at most max files and directories in dirPath.
-	// A max value of -1 returns all files.
-	// If any patterns are passed, then only files or directories with a name that matches
-	// at least one of the patterns are returned.
-	ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) ([]File, error)
-
 	MakeDir(dirPath string, perm []Permissions) error
 
 	OpenReader(filePath string) (ReadCloser, error)
@@ -271,3 +257,19 @@ type MakeAllDirsFileSystem interface {
 
 	MakeAllDirs(dirPath string, perm []Permissions) error
 }
+
+type ListDirMaxFileSystem interface {
+	// ListDirMax returns at most max files and directories in dirPath.
+	// A max value of -1 returns all files.
+	// If any patterns are passed, then only files or directories with a name that matches
+	// at least one of the patterns are returned.
+	ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) ([]File, error)
+}
+
+type ListDirRecursiveFileSystem interface {
+	// ListDirInfoRecursive calls the passed callback function for every file (not directory) in dirPath
+	// recursing into all sub-directories.
+	// If any patterns are passed, then only files (not directories) with a name that matches
+	// at least one of the patterns are returned.
+	ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error
+}

other.go → fsimpl/other.go

@@ -1,4 +1,4 @@
-package fs
+package fsimpl
 
 /*
 import (

listdir.go

@@ -5,21 +5,21 @@ import (
 	"errors"
 )
 
-// ListDirMaxImpl implements the ListDirMax method functionality by calling listDir.
+// listDirMaxImpl implements the ListDirMax method functionality by calling listDir.
 // It returns the passed max number of files or an unlimited number if max is < 0.
 // FileSystem implementations can use this function to implement ListDirMax,
 // if an own, specialized implementation doesn't make sense.
-func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Context, callback func(File) error) error) (files []File, err error) {
+func listDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Context, callback func(File) error) error) (files []File, err error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
 	if max == 0 {
 		return nil, nil
 	}
-	errAbort := errors.New("errAbort") // used as an internal flag, won't be returned
+	done := errors.New("done") // used as an internal flag, won't be returned
 	err = listDir(ctx, func(file File) error {
 		if max >= 0 && len(files) >= max {
-			return errAbort
+			return done
 		}
 		if files == nil {
 			// Reserve space for files
@@ -32,7 +32,7 @@ func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Conte
 		files = append(files, file)
 		return nil
 	})
-	if err != nil && !errors.Is(err, errAbort) {
+	if err != nil && !errors.Is(err, done) {
 		return nil, err
 	}
 	return files, nil
@@ -55,26 +55,3 @@ func ListDirMaxImpl(ctx context.Context, max int, listDir func(ctx context.Conte
 // 		return err
 // 	}, nil)
 // }
-
-// ListDirInfoRecursiveImpl can be used by FileSystem implementations to
-// implement FileSystem.ListDirRecursive if it doesn't have an internal
-// optimzed implementation for it.
-func ListDirInfoRecursiveImpl(ctx context.Context, fs FileSystem, dirPath string, callback func(*FileInfo) error, patterns []string) error {
-	return fs.ListDirInfo(
-		ctx,
-		dirPath,
-		func(info *FileInfo) error {
-			if info.IsDir {
-				err := info.File.ListDirInfoRecursiveContext(ctx, callback, patterns...)
-				// Don't mind files that have been deleted while iterating
-				return RemoveErrDoesNotExist(err)
-			}
-			match, err := fs.MatchAnyPattern(info.Name, patterns)
-			if !match || err != nil {
-				return err
-			}
-			return callback(info)
-		},
-		nil, // No patterns
-	)
-}

listdir_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 )
 
-func TestListDirMaxImpl(t *testing.T) {
+func Test_listDirMaxImpl(t *testing.T) {
 	bg := context.Background()
 	errCtx, cancel := context.WithCancel(context.Background())
 	cancel()
@@ -47,7 +47,7 @@ func TestListDirMaxImpl(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotFiles, err := ListDirMaxImpl(tt.args.ctx, tt.args.max, tt.args.listDir)
+			gotFiles, err := listDirMaxImpl(tt.args.ctx, tt.args.max, tt.args.listDir)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ListDirMaxImpl() error = %v, wantErr %v", err, tt.wantErr)
 				return

localfilesystem.go

@@ -321,15 +321,6 @@ func (local *LocalFileSystem) ListDirInfo(ctx context.Context, dirPath string, c
 	return nil
 }
 
-func (local *LocalFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*FileInfo) error, patterns []string) error {
-	if dirPath == "" {
-		return ErrEmptyPath
-	}
-	dirPath = filepath.Clean(dirPath)
-	dirPath = expandTilde(dirPath)
-	return ListDirInfoRecursiveImpl(ctx, local, dirPath, callback, patterns)
-}
-
 func (local *LocalFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []File, err error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()

multipartfs/multipartfs.go

@@ -254,16 +254,6 @@ func (mpfs *MultipartFileSystem) ListDirInfo(ctx context.Context, dirPath string
 	return err
 }
 
-func (mpfs *MultipartFileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string, callback func(*fs.FileInfo) error, patterns []string) error {
-	return fs.ListDirInfoRecursiveImpl(ctx, mpfs, dirPath, callback, patterns)
-}
-
-func (mpfs *MultipartFileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) {
-	return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error {
-		return mpfs.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns)
-	})
-}
-
 func (mpfs *MultipartFileSystem) ReadAll(ctx context.Context, filePath string) ([]byte, error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()

s3fs/s3fs.go

@@ -281,12 +281,6 @@ func (s *S3FileSystem) ListDirInfoRecursive(ctx context.Context, dirPath string,
 	return s.listDirInfo(ctx, dirPath, callback, patterns, true)
 }
 
-func (s *S3FileSystem) ListDirMax(ctx context.Context, dirPath string, max int, patterns []string) (files []fs.File, err error) {
-	return fs.ListDirMaxImpl(ctx, max, func(ctx context.Context, callback func(fs.File) error) error {
-		return s.ListDirInfo(ctx, dirPath, fs.FileInfoCallback(callback), patterns)
-	})
-}
-
 func (s *S3FileSystem) Touch(filePath string, perm []fs.Permissions) error {
 	if s.Exists(filePath) {
 		return nil // TODO is this OK, can we change the modified time?