From f03f1b01561343a2b1f13207246e3b38a27fcab5 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 8 Apr 2023 10:01:48 +0200 Subject: [PATCH] improve test cases coverage Signed-off-by: Nicola Murino --- go.mod | 48 ++++++++-------- go.sum | 94 ++++++++++++++++---------------- internal/common/connection.go | 4 ++ internal/common/eventmanager.go | 16 ++++-- internal/common/protocol_test.go | 81 +++++++++++++++++++++++---- internal/ftpd/ftpd_test.go | 2 +- internal/httpdtest/httpfsimpl.go | 17 +++++- internal/sftpd/httpfs_test.go | 2 +- internal/vfs/azblobfs.go | 14 ++++- internal/vfs/gcsfs.go | 4 ++ internal/vfs/s3fs.go | 4 ++ 11 files changed, 191 insertions(+), 95 deletions(-) diff --git a/go.mod b/go.mod index 9e7009fce..adcb5f4c4 100644 --- a/go.mod +++ b/go.mod @@ -4,20 +4,20 @@ go 1.20 require ( cloud.google.com/go/storage v1.30.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736 github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964 - github.com/aws/aws-sdk-go-v2 v1.17.7 - github.com/aws/aws-sdk-go-v2/config v1.18.19 - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60 - github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.7 - github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.1 - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 + github.com/aws/aws-sdk-go-v2 v1.17.8 + github.com/aws/aws-sdk-go-v2/config v1.18.20 + github.com/aws/aws-sdk-go-v2/credentials v1.13.19 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 + github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/cockroachdb/cockroach-go/v2 v2.3.3 github.com/coreos/go-oidc/v3 v3.5.0 @@ -68,9 +68,9 @@ require ( go.etcd.io/bbolt v1.3.7 go.uber.org/automaxprocs v1.5.2 gocloud.dev v0.29.0 - golang.org/x/crypto v0.7.0 - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.6.0 + golang.org/x/crypto v0.8.0 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/sys v0.7.0 golang.org/x/term v0.7.0 golang.org/x/time v0.3.0 @@ -86,16 +86,16 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/ajg/form v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect @@ -156,7 +156,7 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect @@ -169,5 +169,5 @@ require ( replace ( github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 github.com/robfig/cron/v3 => github.com/drakkan/cron/v3 v3.0.0-20230222140221-217a1e4d96c0 - golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230305062608-96564146159b + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20230408075646-704a7f627371 ) diff --git a/go.sum b/go.sum index 376de299d..daa419038 100644 --- a/go.sum +++ b/go.sum @@ -431,8 +431,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+Q github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1/go.mod h1:DffdKW9RFqa5VgmsjUOsS7UE7eiA5iAvYUs63bhKQ0M= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0 h1:xGLAFFd9D3iLGxYiUGPdITSzsFmU1K8VtfuUHWAoN7M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 h1:T8quHYlUGyb/oqtSTwqlCr1ilJHrDv+ZtpSfo+hm1BU= @@ -560,67 +560,67 @@ github.com/aws/aws-sdk-go v1.44.187/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/aws/aws-sdk-go v1.44.200/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.8 h1:GMupCNNI7FARX27L7GjCJM8NgivWbRgpjNI/hOQjFS8= +github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.12/go.mod h1:J36fOhj1LQBr+O4hJCiT8FwVvieeoSGOtPuvhKlsNu8= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/config v1.18.20 h1:yYy+onqmLmDVZtx0mkqbx8aJPl+58V6ivLbLDZ2Qztc= +github.com/aws/aws-sdk-go-v2/config v1.18.20/go.mod h1:RWjF39RiDevmHw/+VaD8F0A36OPIPTHQQyRx0eZohnw= github.com/aws/aws-sdk-go-v2/credentials v1.13.12/go.mod h1:37HG2MBroXK3jXfxVGtbM2J48ra2+Ltu+tmwr/jO0KA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/credentials v1.13.19 h1:FWHJy9uggyQCSEhovtl/6W6rW9P6DSr62GUeY/TS6Eo= +github.com/aws/aws-sdk-go-v2/credentials v1.13.19/go.mod h1:2m4uvLvl5hvQezVkLeBBUGMEDm5GcUNc3016W6d3NGg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22/go.mod h1:YGSIJyQ6D6FjKMQh16hVFSIUD54L4F7zTGePqYMYYJU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 h1:jOzQAesnBFDmz93feqKnsTHsXrlwWORNZMFHMV+WLFU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2/go.mod h1:cDh1p6XkSGSwSRIArWRc6+UqAQ7x4alQ0QfpVR6f+co= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51/go.mod h1:7Grl2gV+dx9SWrUIgwwlUvU40t7+lOSbx34XwfmsTkY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60 h1:BRLcU36boYxw6BPOEvwJbDPuCtP7FqMhXMFk2NM6poM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.60/go.mod h1:HLWzCoNyzaPkOOs9yZ3muJ91lSk8O9DJbJw5aKAWWHY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 h1:0fHTNkoMAz7jbXSyo0SLubbTJEO+AgvkZ0iWT9DbEsk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61/go.mod h1:i8l1At/vjpY8xf1ivKUBJE4+DQyE0gM9Zdg3ZpC/7RU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 h1:dpbVNUjczQ8Ae3QKHbpHBpfvaVkRdesxpTOe9pTouhU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32/go.mod h1:RudqOgadTWdcS3t/erPQo24pcVEoYyqj/kKW5Vya21I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 h1:QH2kOS3Ht7x+u0gHCh06CXL/h6G8LQJFpZfFBYBNboo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26/go.mod h1:vq86l7956VgFr0/FWQ2BWnK07QC3WYsepKzy33qqY5U= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 h1:HbH1VjUgrCdLJ+4lnnuLI4iVNRvBbBELGaJ5f69ClA8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33/go.mod h1:zG2FcwjQarWaqXSCGpgcr3RSjZ6dHGguZSppUL0XR7Q= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19/go.mod h1:8W88sW3PjamQpKFUQvHWWKay6ARsNvZnzU7+a4apubw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 h1:DWYZIsyqagnWL00f8M/SOr9fN063OEQWn9LLTbdYXsk= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23/go.mod h1:uIiFgURZbACBEQJfqTZPb/jxO7R+9LeoHUFudtIdeQI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 h1:zsg+5ouVLLbePknVZlUMm1ptwyQLkjjLMWnN+kVs5dA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24/go.mod h1:+fFaIjycTmpV6hjmPTbyU9Kp5MI/lA+bbibcAtmlhYA= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23/go.mod h1:1jcUfF+FAOEwtIcNiHPaV4TSoZqkUIPzrohmD7fb95c= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 h1:CeuSeq/8FnYpPtnuIeLQEEvDv9zUjneuYi8EghMBdwQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26/go.mod h1:2UqAAwMUXKeRkAHIlDJqvMVgOWkUi/AUXPk/YIe+Dg4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27 h1:qIw7Hg5eJEc1uSxg3hRwAthPAO7NeOd4dPxhaTi0yB0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27/go.mod h1:Zz0kvhcSlu3NX4XJkaGgdjaa+u7a9LYuy8JKxA5v3RM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22/go.mod h1:xt0Au8yPIwYXf/GYPy/vl4K3CgwhfQMYbrH7DlUUIws= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 h1:uUt4XctZLhl9wBE1L8lobU3bVN8SNUP7T+olb0bWBO4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26/go.mod h1:Bd4C/4PkVGubtNe5iMXu5BNnaBi/9t/UsFspPt4ram8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22/go.mod h1:QFVbqK54XArazLvn2wvWMRBi/jGrWii46qbr5DyPGjc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 h1:e2ooMhpYGhDnBfSvIyusvAwX7KexuZaHbQY2Dyei7VU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0/go.mod h1:bh2E0CXKZsQN+faiKVqC40vfNMAWheoULBCnEgO9K+8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 h1:lRWp3bNu5wy0X3a8GS42JvZFlv++AKsMdzEnoiVJrkg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1/go.mod h1:VXBHSxdN46bsJrkniN68psSwbyBKsazQfU2yX/iSDso= github.com/aws/aws-sdk-go-v2/service/kms v1.20.2/go.mod h1:vdqtUOdVuf5ooy+hJ2GnzqNo94xiAA9s1xbZ1hQgRE0= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.7 h1:QA+w3BlShNMgOTRN1kBG2qOIHuTzVTxZ0l3ImKkz+ic= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.7/go.mod h1:8Ma9cuRQADU+FwLPYmoSdvUt4tItX2kleWi/++H0BF0= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.8 h1:NLpW2zjplPmay81ZU6SbFZc6oI156PxFvU/zn41WlQE= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.14.8/go.mod h1:oNFgSKrV6y06CQnFXzcYtEOB/EfRP1aHWbddn1TEXG8= github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2/go.mod h1:SXDHd6fI2RhqB7vmAzyYQCTQnpZrIprVJvYxpzW3JAM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 h1:PJH4I+qYjPXclKRbVCW47iYUvtXEh1u6YmDhn5J8VQE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1/go.mod h1:ncltU6n4Nof5uJttDtcNQ537uNuwYqsZZQcpkd2/GUQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 h1:iOZoYePk+EuBI1tC7bxeRjO+JvClcYm2fZYW5WPIOMQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2/go.mod h1:aSl9/LJltSz1cVusiR/Mu8tvI4Sv/5w/WWrJmmkNii0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.18.3/go.mod h1:hqPcyOuLU6yWIbLy3qMnQnmidgKuIEwqIlW6+chYnog= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.1 h1:+rANS0SbrDUqF3VJeil1HJHhNK8vdUu1VGqnkr4o6kw= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.1/go.mod h1:SUiYnlcBDUvSLD6iUmwSwXni2i6iGa9WHc+eM5061W4= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.2 h1:mRA8bnA0zdTvsGXmoZ6EOmTTmORjEV1uareB4GfzfK0= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.2/go.mod h1:QNYziZIPDbKmKRoTHi9wkgqVidknyiGHfig1UNOojqk= github.com/aws/aws-sdk-go-v2/service/sns v1.20.2/go.mod h1:VN2n9SOMS1lNbh5YD7o+ho0/rgfifSrK//YYNiVVF5E= github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2/go.mod h1:1ttxGjUHZliCQMpPss1sU5+Ph/5NvdMFRzr96bv8gm0= github.com/aws/aws-sdk-go-v2/service/ssm v1.35.2/go.mod h1:VLSz2SHUKYFSOlXB/GlXoLU6KPYQJAbw7I20TDJdyws= github.com/aws/aws-sdk-go-v2/service/sso v1.12.1/go.mod h1:IgV8l3sj22nQDd5qcAGY0WenwCzCphqdbFOpfktZPrI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 h1:rrYYhsvcvg6CDDoo4GHKtAWBFutS86CpmGvqHJHYL9w= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.7/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1/go.mod h1:O1YSOg3aekZibh2SngvCRRG+cRHKKlYgxf/JBF/Kr/k= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 h1:Vjpjt3svuJ/u+eKRfycZwqLsLoxyuvvZyHMJSk+3k58= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiOVVG8uVjGI6HaZ8WBHdgDgU= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 h1:SQ8pPoXfzuz4DImO4KAi9xhO4ANG0Ckb5clZ5GoRAPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.8/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -876,8 +876,8 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/drakkan/cron/v3 v3.0.0-20230222140221-217a1e4d96c0 h1:EW9gIJRmt9lzk66Fhh4S8VEtURA6QHZqGeSRE9Nb2/U= github.com/drakkan/cron/v3 v3.0.0-20230222140221-217a1e4d96c0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/drakkan/crypto v0.0.0-20230305062608-96564146159b h1:wnjew0eGed6jB+NRjx45Wa/u2yMqokQrw2XmVORY9xI= -github.com/drakkan/crypto v0.0.0-20230305062608-96564146159b/go.mod h1:Hupq8gBhTW/W5Zsg6pKsZ+qnw9ZjHlt/7qnltlpCV7I= +github.com/drakkan/crypto v0.0.0-20230408075646-704a7f627371 h1:e2fWtTFAkFfNOeqww6HsEhtxETjGUBKnmIbMNB7V8mg= +github.com/drakkan/crypto v0.0.0-20230408075646-704a7f627371/go.mod h1:svd5Kbdx1UEmxh6mV0H38ASBeI90vEuujcyP74bw210= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= github.com/drakkan/webdav v0.0.0-20230227175313-32996838bcd8 h1:tdkLkSKtYd3WSDsZXGJDKsakiNstLQJPN5HjnqCkf2c= @@ -2243,8 +2243,8 @@ golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmL golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2276,8 +2276,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2460,7 +2460,6 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2477,7 +2476,6 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2583,8 +2581,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/common/connection.go b/internal/common/connection.go index 8d0629b2d..5f1b7630f 100644 --- a/internal/common/connection.go +++ b/internal/common/connection.go @@ -650,6 +650,10 @@ func (c *BaseConnection) doRecursiveCopy(virtualSourcePath, virtualTargetPath st return err } if err := c.doRecursiveCopy(sourcePath, targetPath, info, true); err != nil { + if c.IsNotExistError(err) { + c.Log(logger.LevelInfo, "skipping copy for source path %q: %v", sourcePath, err) + continue + } return err } } diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index 66b594207..ef6531c75 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -816,22 +816,28 @@ func getCSVRetentionReport(results []folderRetentionCheckResult) ([]byte, error) func closeWriterAndUpdateQuota(w io.WriteCloser, conn *BaseConnection, virtualSourcePath, virtualTargetPath string, numFiles int, truncatedSize int64, errTransfer error, operation string, startTime time.Time, ) error { + var fsDstPath string + var errDstFs error errWrite := w.Close() targetPath := virtualSourcePath if virtualTargetPath != "" { targetPath = virtualTargetPath + var fsDst vfs.Fs + fsDst, fsDstPath, errDstFs = conn.GetFsAndResolvedPath(virtualTargetPath) + if errTransfer != nil && errDstFs == nil { + // try to remove a partial file on error. If this fails, we can't do anything + errRemove := fsDst.Remove(fsDstPath, false) + conn.Log(logger.LevelDebug, "removing partial file %q after write error, result: %v", virtualTargetPath, errRemove) + } } info, err := conn.doStatInternal(targetPath, 0, false, false) if err == nil { updateUserQuotaAfterFileWrite(conn, targetPath, numFiles, info.Size()-truncatedSize) - var fsSrcPath, fsDstPath string - var errSrcFs, errDstFs error + var fsSrcPath string + var errSrcFs error if virtualSourcePath != "" { _, fsSrcPath, errSrcFs = conn.GetFsAndResolvedPath(virtualSourcePath) } - if virtualTargetPath != "" { - _, fsDstPath, errDstFs = conn.GetFsAndResolvedPath(virtualTargetPath) - } if errSrcFs == nil && errDstFs == nil { elapsed := time.Since(startTime).Nanoseconds() / 1000000 if errTransfer == nil { diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 948b88bf9..b292087cd 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -67,17 +67,20 @@ import ( ) const ( - httpAddr = "127.0.0.1:9999" - httpProxyAddr = "127.0.0.1:7777" - sftpServerAddr = "127.0.0.1:4022" - smtpServerAddr = "127.0.0.1:2525" - webDavServerPort = 9191 - defaultUsername = "test_common_sftp" - defaultPassword = "test_password" - defaultSFTPUsername = "test_common_sftpfs_user" - osWindows = "windows" - testFileName = "test_file_common_sftp.dat" - testDir = "test_dir_common" + httpAddr = "127.0.0.1:9999" + httpProxyAddr = "127.0.0.1:7777" + sftpServerAddr = "127.0.0.1:4022" + smtpServerAddr = "127.0.0.1:2525" + webDavServerPort = 9191 + httpFsPort = 34567 + defaultUsername = "test_common_sftp" + defaultPassword = "test_password" + defaultSFTPUsername = "test_common_sftpfs_user" + defaultHTTPFsUsername = "httpfs_ftp_user" + httpFsWellKnowDir = "/wellknow" + osWindows = "windows" + testFileName = "test_file_common_sftp.dat" + testDir = "test_dir_common" ) var ( @@ -181,6 +184,7 @@ func TestMain(m *testing.M) { waitTCPListening(sftpdConf.Bindings[0].GetAddress()) waitTCPListening(httpdConf.Bindings[0].GetAddress()) waitTCPListening(webDavConf.Bindings[0].GetAddress()) + startHTTPFs() go func() { // start a test HTTP server to receive action notifications @@ -8037,6 +8041,30 @@ func TestCrossFoldersCopy(t *testing.T) { } } +func TestHTTPFs(t *testing.T) { + u := getTestUserWithHTTPFs() + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + + err = os.MkdirAll(user.GetHomeDir(), os.ModePerm) + assert.NoError(t, err) + + conn := common.NewBaseConnection(xid.New().String(), common.ProtocolFTP, "", "", user) + err = conn.CreateDir(httpFsWellKnowDir, false) + assert.NoError(t, err) + + err = os.WriteFile(filepath.Join(os.TempDir(), "httpfs", defaultHTTPFsUsername, httpFsWellKnowDir, "file.txt"), []byte("data"), 0666) + assert.NoError(t, err) + + err = conn.Copy(httpFsWellKnowDir, httpFsWellKnowDir+"_copy") + assert.NoError(t, err) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + func TestProxyProtocol(t *testing.T) { resp, err := httpclient.Get(fmt.Sprintf("http://%v", httpProxyAddr)) if assert.NoError(t, err) { @@ -8210,6 +8238,18 @@ func getCryptFsUser() dataprovider.User { return u } +func getTestUserWithHTTPFs() dataprovider.User { + u := getTestUser() + u.FsConfig.Provider = sdk.HTTPFilesystemProvider + u.FsConfig.HTTPConfig = vfs.HTTPFsConfig{ + BaseHTTPFsConfig: sdk.BaseHTTPFsConfig{ + Endpoint: fmt.Sprintf("http://127.0.0.1:%d/api/v1", httpFsPort), + Username: defaultHTTPFsUsername, + }, + } + return u +} + func writeSFTPFile(name string, size int64, client *sftp.Client) error { err := writeSFTPFileNoCheck(name, size, client) if err != nil { @@ -8345,3 +8385,22 @@ func (e *receivedEmail) get() receivedEmail { Data: e.Data, } } + +func startHTTPFs() { + go func() { + readdirCallback := func(name string) []os.FileInfo { + if name == httpFsWellKnowDir { + return []os.FileInfo{vfs.NewFileInfo("ghost.txt", false, 0, time.Unix(0, 0), false)} + } + return nil + } + callbacks := &httpdtest.HTTPFsCallbacks{ + Readdir: readdirCallback, + } + if err := httpdtest.StartTestHTTPFs(httpFsPort, callbacks); err != nil { + logger.ErrorToConsole("could not start HTTPfs test server: %v", err) + os.Exit(1) + } + }() + waitTCPListening(fmt.Sprintf(":%d", httpFsPort)) +} diff --git a/internal/ftpd/ftpd_test.go b/internal/ftpd/ftpd_test.go index e32a71015..cc935e63a 100644 --- a/internal/ftpd/ftpd_test.go +++ b/internal/ftpd/ftpd_test.go @@ -4011,7 +4011,7 @@ func generateTOTPPasscode(secret string, algo otp.Algorithm) (string, error) { func startHTTPFs() { go func() { - if err := httpdtest.StartTestHTTPFs(httpFsPort); err != nil { + if err := httpdtest.StartTestHTTPFs(httpFsPort, nil); err != nil { logger.ErrorToConsole("could not start HTTPfs test server: %v", err) os.Exit(1) } diff --git a/internal/httpdtest/httpfsimpl.go b/internal/httpdtest/httpfsimpl.go index fd8bb3902..4be312024 100644 --- a/internal/httpdtest/httpfsimpl.go +++ b/internal/httpdtest/httpfsimpl.go @@ -52,12 +52,19 @@ const ( statvfsPath = "/api/v1/statvfs" ) +// HTTPFsCallbacks defines additional callbacks to customize the HTTPfs responses +type HTTPFsCallbacks struct { + Readdir func(string) []os.FileInfo +} + // StartTestHTTPFs starts a test HTTP service that implements httpfs // and listens on the specified port -func StartTestHTTPFs(port int) error { +func StartTestHTTPFs(port int, callbacks *HTTPFsCallbacks) error { fs := httpFsImpl{ - port: port, + port: port, + callbacks: callbacks, } + return fs.Run() } @@ -75,6 +82,7 @@ type httpFsImpl struct { basePath string port int unixSocketPath string + callbacks *HTTPFsCallbacks } type apiResponse struct { @@ -358,6 +366,11 @@ func (fs *httpFsImpl) readdir(w http.ResponseWriter, r *http.Request) { for _, fi := range list { result = append(result, getStatFromInfo(fi)) } + if fs.callbacks != nil && fs.callbacks.Readdir != nil { + for _, fi := range fs.callbacks.Readdir(name) { + result = append(result, getStatFromInfo(fi)) + } + } render.JSON(w, r, result) } diff --git a/internal/sftpd/httpfs_test.go b/internal/sftpd/httpfs_test.go index 95d669895..41c3faedb 100644 --- a/internal/sftpd/httpfs_test.go +++ b/internal/sftpd/httpfs_test.go @@ -358,7 +358,7 @@ func startHTTPFs() { }() } go func() { - if err := httpdtest.StartTestHTTPFs(httpFsPort); err != nil { + if err := httpdtest.StartTestHTTPFs(httpFsPort, nil); err != nil { logger.ErrorToConsole("could not start HTTPfs test server: %v", err) os.Exit(1) } diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 34809aafa..739d807be 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -859,6 +859,10 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (in targetEntry := fs.Join(target, info.Name()) files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) if err != nil { + if fs.IsNotExist(err) { + fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err) + continue + } return numFiles, filesSize, err } numFiles += files @@ -882,11 +886,15 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo) (in } } } - err := fs.Remove(source, fi.IsDir()) + err := fs.skipNotExistErr(fs.Remove(source, fi.IsDir())) + return numFiles, filesSize, err +} + +func (fs *AzureBlobFs) skipNotExistErr(err error) error { if fs.IsNotExist(err) { - err = nil + return nil } - return numFiles, filesSize, err + return err } func (fs *AzureBlobFs) mkdirInternal(name string) error { diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index cf2b2aef7..016e55611 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -801,6 +801,10 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo) (int, int targetEntry := fs.Join(target, info.Name()) files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) if err != nil { + if fs.IsNotExist(err) { + fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err) + continue + } return numFiles, filesSize, err } numFiles += files diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index e48195ac9..970fc45ab 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -781,6 +781,10 @@ func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo) (int, int6 targetEntry := fs.Join(target, info.Name()) files, size, err := fs.renameInternal(sourceEntry, targetEntry, info) if err != nil { + if fs.IsNotExist(err) { + fsLog(fs, logger.LevelInfo, "skipping rename for %q: %v", sourceEntry, err) + continue + } return numFiles, filesSize, err } numFiles += files