improve test cases coverage

Signed-off-by: Nicola Murino <[email protected]>

go.mod

@@ -4,20 +4,20 @@ go 1.20
 
 require (
 	cloud.google.com/go/storage v1.30.1
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5
 	github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736
 	github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964
-	github.com/aws/aws-sdk-go-v2 v1.17.7
-	github.com/aws/aws-sdk-go-v2/config v1.18.19
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.18
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60
-	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.7
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1
-	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.1
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.7
+	github.com/aws/aws-sdk-go-v2 v1.17.8
+	github.com/aws/aws-sdk-go-v2/config v1.18.20
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.19
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61
+	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.8
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.2
+	github.com/aws/aws-sdk-go-v2/service/sts v1.18.8
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/cockroachdb/cockroach-go/v2 v2.3.3
 	github.com/coreos/go-oidc/v3 v3.5.0
@@ -68,9 +68,9 @@ require (
 	go.etcd.io/bbolt v1.3.7
 	go.uber.org/automaxprocs v1.5.2
 	gocloud.dev v0.29.0
-	golang.org/x/crypto v0.7.0
-	golang.org/x/net v0.8.0
-	golang.org/x/oauth2 v0.6.0
+	golang.org/x/crypto v0.8.0
+	golang.org/x/net v0.9.0
+	golang.org/x/oauth2 v0.7.0
 	golang.org/x/sys v0.7.0
 	golang.org/x/term v0.7.0
 	golang.org/x/time v0.3.0
@@ -86,16 +86,16 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1 // indirect
@@ -156,7 +156,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.8.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect
@@ -169,5 +169,5 @@ require (
 replace (
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
 	github.com/robfig/cron/v3 => github.com/drakkan/cron/v3 v3.0.0-20230222140221-217a1e4d96c0
-	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230305062608-96564146159b
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230408075646-704a7f627371
 )

internal/common/connection.go

@@ -650,6 +650,10 @@ func (c *BaseConnection) doRecursiveCopy(virtualSourcePath, virtualTargetPath st
 				return err
 			}
 			if err := c.doRecursiveCopy(sourcePath, targetPath, info, true); err != nil {
+				if c.IsNotExistError(err) {
+					c.Log(logger.LevelInfo, "skipping copy for source path %q: %v", sourcePath, err)
+					continue
+				}
 				return err
 			}
 		}

internal/common/eventmanager.go

@@ -816,22 +816,28 @@ func getCSVRetentionReport(results []folderRetentionCheckResult) ([]byte, error)
 func closeWriterAndUpdateQuota(w io.WriteCloser, conn *BaseConnection, virtualSourcePath, virtualTargetPath string,
 	numFiles int, truncatedSize int64, errTransfer error, operation string, startTime time.Time,
 ) error {
+	var fsDstPath string
+	var errDstFs error
 	errWrite := w.Close()
 	targetPath := virtualSourcePath
 	if virtualTargetPath != "" {
 		targetPath = virtualTargetPath
+		var fsDst vfs.Fs
+		fsDst, fsDstPath, errDstFs = conn.GetFsAndResolvedPath(virtualTargetPath)
+		if errTransfer != nil && errDstFs == nil {
+			// try to remove a partial file on error. If this fails, we can't do anything
+			errRemove := fsDst.Remove(fsDstPath, false)
+			conn.Log(logger.LevelDebug, "removing partial file %q after write error, result: %v", virtualTargetPath, errRemove)
+		}
 	}
 	info, err := conn.doStatInternal(targetPath, 0, false, false)
 	if err == nil {
 		updateUserQuotaAfterFileWrite(conn, targetPath, numFiles, info.Size()-truncatedSize)
-		var fsSrcPath, fsDstPath string
-		var errSrcFs, errDstFs error
+		var fsSrcPath string
+		var errSrcFs error
 		if virtualSourcePath != "" {
 			_, fsSrcPath, errSrcFs = conn.GetFsAndResolvedPath(virtualSourcePath)
 		}
-		if virtualTargetPath != "" {
-			_, fsDstPath, errDstFs = conn.GetFsAndResolvedPath(virtualTargetPath)
-		}
 		if errSrcFs == nil && errDstFs == nil {
 			elapsed := time.Since(startTime).Nanoseconds() / 1000000
 			if errTransfer == nil {

internal/common/protocol_test.go

@@ -67,17 +67,20 @@ import (
 )
 
 const (
-	httpAddr            = "127.0.0.1:9999"
-	httpProxyAddr       = "127.0.0.1:7777"
-	sftpServerAddr      = "127.0.0.1:4022"
-	smtpServerAddr      = "127.0.0.1:2525"
-	webDavServerPort    = 9191
-	defaultUsername     = "test_common_sftp"
-	defaultPassword     = "test_password"
-	defaultSFTPUsername = "test_common_sftpfs_user"
-	osWindows           = "windows"
-	testFileName        = "test_file_common_sftp.dat"
-	testDir             = "test_dir_common"
+	httpAddr              = "127.0.0.1:9999"
+	httpProxyAddr         = "127.0.0.1:7777"
+	sftpServerAddr        = "127.0.0.1:4022"
+	smtpServerAddr        = "127.0.0.1:2525"
+	webDavServerPort      = 9191
+	httpFsPort            = 34567
+	defaultUsername       = "test_common_sftp"
+	defaultPassword       = "test_password"
+	defaultSFTPUsername   = "test_common_sftpfs_user"
+	defaultHTTPFsUsername = "httpfs_ftp_user"
+	httpFsWellKnowDir     = "/wellknow"
+	osWindows             = "windows"
+	testFileName          = "test_file_common_sftp.dat"
+	testDir               = "test_dir_common"
 )
 
 var (
@@ -181,6 +184,7 @@ func TestMain(m *testing.M) {
 	waitTCPListening(sftpdConf.Bindings[0].GetAddress())
 	waitTCPListening(httpdConf.Bindings[0].GetAddress())
 	waitTCPListening(webDavConf.Bindings[0].GetAddress())
+	startHTTPFs()
 
 	go func() {
 		// start a test HTTP server to receive action notifications
@@ -8037,6 +8041,30 @@ func TestCrossFoldersCopy(t *testing.T) {
 	}
 }
 
+func TestHTTPFs(t *testing.T) {
+	u := getTestUserWithHTTPFs()
+	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+
+	err = os.MkdirAll(user.GetHomeDir(), os.ModePerm)
+	assert.NoError(t, err)
+
+	conn := common.NewBaseConnection(xid.New().String(), common.ProtocolFTP, "", "", user)
+	err = conn.CreateDir(httpFsWellKnowDir, false)
+	assert.NoError(t, err)
+
+	err = os.WriteFile(filepath.Join(os.TempDir(), "httpfs", defaultHTTPFsUsername, httpFsWellKnowDir, "file.txt"), []byte("data"), 0666)
+	assert.NoError(t, err)
+
+	err = conn.Copy(httpFsWellKnowDir, httpFsWellKnowDir+"_copy")
+	assert.NoError(t, err)
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestProxyProtocol(t *testing.T) {
 	resp, err := httpclient.Get(fmt.Sprintf("http://%v", httpProxyAddr))
 	if assert.NoError(t, err) {
@@ -8210,6 +8238,18 @@ func getCryptFsUser() dataprovider.User {
 	return u
 }
 
+func getTestUserWithHTTPFs() dataprovider.User {
+	u := getTestUser()
+	u.FsConfig.Provider = sdk.HTTPFilesystemProvider
+	u.FsConfig.HTTPConfig = vfs.HTTPFsConfig{
+		BaseHTTPFsConfig: sdk.BaseHTTPFsConfig{
+			Endpoint: fmt.Sprintf("http://127.0.0.1:%d/api/v1", httpFsPort),
+			Username: defaultHTTPFsUsername,
+		},
+	}
+	return u
+}
+
 func writeSFTPFile(name string, size int64, client *sftp.Client) error {
 	err := writeSFTPFileNoCheck(name, size, client)
 	if err != nil {
@@ -8345,3 +8385,22 @@ func (e *receivedEmail) get() receivedEmail {
 		Data: e.Data,
 	}
 }
+
+func startHTTPFs() {
+	go func() {
+		readdirCallback := func(name string) []os.FileInfo {
+			if name == httpFsWellKnowDir {
+				return []os.FileInfo{vfs.NewFileInfo("ghost.txt", false, 0, time.Unix(0, 0), false)}
+			}
+			return nil
+		}
+		callbacks := &httpdtest.HTTPFsCallbacks{
+			Readdir: readdirCallback,
+		}
+		if err := httpdtest.StartTestHTTPFs(httpFsPort, callbacks); err != nil {
+			logger.ErrorToConsole("could not start HTTPfs test server: %v", err)
+			os.Exit(1)
+		}
+	}()
+	waitTCPListening(fmt.Sprintf(":%d", httpFsPort))
+}

internal/ftpd/ftpd_test.go

@@ -4011,7 +4011,7 @@ func generateTOTPPasscode(secret string, algo otp.Algorithm) (string, error) {
 
 func startHTTPFs() {
 	go func() {
-		if err := httpdtest.StartTestHTTPFs(httpFsPort); err != nil {
+		if err := httpdtest.StartTestHTTPFs(httpFsPort, nil); err != nil {
 			logger.ErrorToConsole("could not start HTTPfs test server: %v", err)
 			os.Exit(1)
 		}

internal/httpdtest/httpfsimpl.go

@@ -52,12 +52,19 @@ const (
 	statvfsPath  = "/api/v1/statvfs"
 )
 
+// HTTPFsCallbacks defines additional callbacks to customize the HTTPfs responses
+type HTTPFsCallbacks struct {
+	Readdir func(string) []os.FileInfo
+}
+
 // StartTestHTTPFs starts a test HTTP service that implements httpfs
 // and listens on the specified port
-func StartTestHTTPFs(port int) error {
+func StartTestHTTPFs(port int, callbacks *HTTPFsCallbacks) error {
 	fs := httpFsImpl{
-		port: port,
+		port:      port,
+		callbacks: callbacks,
 	}
+
 	return fs.Run()
 }
 
@@ -75,6 +82,7 @@ type httpFsImpl struct {
 	basePath       string
 	port           int
 	unixSocketPath string
+	callbacks      *HTTPFsCallbacks
 }
 
 type apiResponse struct {
@@ -358,6 +366,11 @@ func (fs *httpFsImpl) readdir(w http.ResponseWriter, r *http.Request) {
 	for _, fi := range list {
 		result = append(result, getStatFromInfo(fi))
 	}
+	if fs.callbacks != nil && fs.callbacks.Readdir != nil {
+		for _, fi := range fs.callbacks.Readdir(name) {
+			result = append(result, getStatFromInfo(fi))
+		}
+	}
 	render.JSON(w, r, result)
 }
 

internal/sftpd/httpfs_test.go

@@ -358,7 +358,7 @@ func startHTTPFs() {
 		}()
 	}
 	go func() {
-		if err := httpdtest.StartTestHTTPFs(httpFsPort); err != nil {
+		if err := httpdtest.StartTestHTTPFs(httpFsPort, nil); err != nil {
 			logger.ErrorToConsole("could not start HTTPfs test server: %v", err)
 			os.Exit(1)
 		}

internal/vfs/azblobfs.go

@@ -859,6 +859,10 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (in
 				targetEntry := fs.Join(target, info.Name())
 				files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
 				if err != nil {
+					if fs.IsNotExist(err) {
+						fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err)
+						continue
+					}
 					return numFiles, filesSize, err
 				}
 				numFiles += files
@@ -882,11 +886,15 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (in
 			}
 		}
 	}
-	err := fs.Remove(source, fi.IsDir())
+	err := fs.skipNotExistErr(fs.Remove(source, fi.IsDir()))
+	return numFiles, filesSize, err
+}
+
+func (fs *AzureBlobFs) skipNotExistErr(err error) error {
 	if fs.IsNotExist(err) {
-		err = nil
+		return nil
 	}
-	return numFiles, filesSize, err
+	return err
 }
 
 func (fs *AzureBlobFs) mkdirInternal(name string) error {

internal/vfs/gcsfs.go

@@ -801,6 +801,10 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo) (int, int
 				targetEntry := fs.Join(target, info.Name())
 				files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
 				if err != nil {
+					if fs.IsNotExist(err) {
+						fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err)
+						continue
+					}
 					return numFiles, filesSize, err
 				}
 				numFiles += files

internal/vfs/s3fs.go

@@ -781,6 +781,10 @@ func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo) (int, int6
 				targetEntry := fs.Join(target, info.Name())
 				files, size, err := fs.renameInternal(sourceEntry, targetEntry, info)
 				if err != nil {
+					if fs.IsNotExist(err) {
+						fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err)
+						continue
+					}
 					return numFiles, filesSize, err
 				}
 				numFiles += files