Ensure we run scoped tests + added more coverage for removing empty f…

…olders

src/Foundatio.AWS/Extensions/Extensions.cs

@@ -19,10 +19,6 @@ internal static FileSpec ToFileInfo(this S3Object blob)
             if (blob == null)
                 return null;
 
-            // Skip directories
-            if (blob.Key is not null && blob.Size is 0 && blob.Key.EndsWith("/"))
-                return null;
-
             return new FileSpec
             {
                 Path = blob.Key,
@@ -43,5 +39,10 @@ internal static IEnumerable<S3Object> MatchesPattern(this IEnumerable<S3Object>
                 return patternRegex == null || patternRegex.IsMatch(info.Path);
             });
         }
+
+        internal static bool IsDirectory(this FileSpec file)
+        {
+            return file.Path is not null && file.Size is 0 && file.Path.EndsWith("/");
+        }
     }
 }

src/Foundatio.AWS/Storage/S3FileStorage.cs

@@ -361,7 +361,7 @@ private async Task<NextPageResult> GetFiles(SearchCriteria criteria, int pageSiz
             {
                 Success = response.HttpStatusCode.IsSuccessful(),
                 HasMore = response.IsTruncated,
-                Files = response.S3Objects.MatchesPattern(criteria.Pattern).Select(blob => blob.ToFileInfo()).ToList(),
+                Files = response.S3Objects.MatchesPattern(criteria.Pattern).Select(blob => blob.ToFileInfo()).Where(spec => spec is not null && !spec.IsDirectory()).ToList(),
                 NextPageFunc = response.IsTruncated ? _ => GetFiles(criteria, pageSize, cancellationToken, response.NextContinuationToken) : null
             };
         }

tests/Foundatio.AWS.Tests/Storage/S3FileStorageTests.cs

@@ -153,10 +153,11 @@ public virtual async Task WillNotReturnDirectoryInGetPagedFileListAsync()
                 var client = storage is S3FileStorage s3Storage ? s3Storage.Client : null;
                 Assert.NotNull(client);
 
+                const string folderName = "EmptyFolder/";
                 await client.PutObjectAsync(new PutObjectRequest
                 {
                     BucketName = BUCKET_NAME,
-                    Key = "EmptyFolder/",
+                    Key = folderName,
                     ContentBody = String.Empty
                 });
 
@@ -166,6 +167,16 @@ await client.PutObjectAsync(new PutObjectRequest
                 Assert.False(await result.NextPageAsync());
                 Assert.False(result.HasMore);
                 Assert.Empty(result.Files);
+
+                // Ensure the file can be returned via get file info
+                var info = await storage.GetFileInfoAsync(folderName);
+                Assert.NotNull(info?.Path);
+
+                // Ensure delete files can remove all files including fake folders
+                await storage.DeleteFilesAsync("*");
+
+                info = await storage.GetFileInfoAsync(folderName);
+                Assert.Null(info);
             }
         }
 

tests/Foundatio.AWS.Tests/Storage/ScopedS3StorageTests.cs

@@ -1,7 +1,9 @@
-using System.Threading.Tasks;
+using System;
+using System.Threading.Tasks;
 using Amazon;
 using Amazon.Runtime;
 using Amazon.S3;
+using Amazon.S3.Model;
 using Foundatio.Storage;
 using Foundatio.Tests.Storage;
 using Xunit;
@@ -11,13 +13,15 @@ namespace Foundatio.AWS.Tests.Storage
 {
     public class ScopedS3StorageTests : FileStorageTestsBase
     {
+        private const string BUCKET_NAME = "foundatio-ci";
+
         public ScopedS3StorageTests(ITestOutputHelper output) : base(output) { }
 
         protected override IFileStorage GetStorage()
         {
-            return new S3FileStorage(
-                o => o.ConnectionString($"serviceurl=http://localhost:4566;bucket=foundatio-ci;AccessKey=xxx;SecretKey=xxx")
-                    .LoggerFactory(Log));
+            return new ScopedFileStorage(new S3FileStorage(
+                o => o.ConnectionString($"serviceurl=http://localhost:4566;bucket={BUCKET_NAME};AccessKey=xxx;SecretKey=xxx")
+                    .LoggerFactory(Log)), "scoped");
         }
 
         [Fact]
@@ -110,17 +114,64 @@ public override Task CanDeleteSpecificFilesInNestedFolderAsync()
             return base.CanDeleteSpecificFilesInNestedFolderAsync();
         }
 
-        protected override async Task ResetAsync(IFileStorage storage)
-        {
-            var client = new AmazonS3Client(
-                new BasicAWSCredentials("xxx", "xxx"),
-                new AmazonS3Config
+        [Fact]
+        public virtual async Task WillNotReturnDirectoryInGetPagedFileListAsync()
+        {
+            var storage = GetStorage();
+            if (storage == null)
+                return;
+
+            await ResetAsync(storage);
+
+            using (storage)
+            {
+                var result = await storage.GetPagedFileListAsync();
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+                Assert.False(await result.NextPageAsync());
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+
+                // To create an empty folder (or what appears as a folder) in an Amazon S3 bucket using the AWS SDK for .NET,
+                // you typically create an object with a key that ends with a trailing slash ('/') because S3 doesn't
+                // actually have a concept of folders, but it mimics the behavior of folders using object keys.
+                var client = storage is ScopedFileStorage { UnscopedStorage: S3FileStorage s3FileStorage }
+                    ? s3FileStorage.Client
+                    : null;
+                Assert.NotNull(client);
+
+                const string folderName = "EmptyFolder/";
+                await client.PutObjectAsync(new PutObjectRequest
                 {
-                    RegionEndpoint = RegionEndpoint.USEast1,
-                    ServiceURL = "http://localhost:4566",
-                    ForcePathStyle = true
+                    BucketName = BUCKET_NAME,
+                    Key = String.Concat("scoped/", folderName),
+                    ContentBody = String.Empty
                 });
-            await client.PutBucketAsync("foundatio-ci");
+
+                result = await storage.GetPagedFileListAsync();
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+                Assert.False(await result.NextPageAsync());
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+
+                // Ensure the file can be returned via get file info
+                var info = await storage.GetFileInfoAsync(folderName);
+                Assert.NotNull(info?.Path);
+
+                // Ensure delete files can remove all files including fake folders
+                await storage.DeleteFilesAsync("*");
+
+                info = await storage.GetFileInfoAsync(folderName);
+                Assert.Null(info);
+            }
+        }
+
+        protected override async Task ResetAsync(IFileStorage storage)
+        {
+            var client = storage is ScopedFileStorage { UnscopedStorage: S3FileStorage s3FileStorage } ? s3FileStorage.Client : null;
+            Assert.NotNull(client);
+            await client.PutBucketAsync(BUCKET_NAME);
 
             await base.ResetAsync(storage);
         }