From c83a6a5a721d6ff51d7f6cd7652717bdc264dd21 Mon Sep 17 00:00:00 2001
From: Blake Niemyjski <bniemyjski@gmail.com>
Date: Wed, 13 Mar 2024 16:03:31 -0500
Subject: [PATCH] Added support to not return virtual folders in get files
 implementation.

---
 src/Foundatio.AWS/Extensions/Extensions.cs    | 13 ++++-
 .../Metrics/CloudWatchMetricsClient.cs        |  1 -
 src/Foundatio.AWS/Queues/SQSQueueEntry.cs     |  1 -
 .../Storage/S3FileStorageTests.cs             | 58 +++++++++++++++----
 4 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/src/Foundatio.AWS/Extensions/Extensions.cs b/src/Foundatio.AWS/Extensions/Extensions.cs
index 748ae4f..2ce104b 100644
--- a/src/Foundatio.AWS/Extensions/Extensions.cs
+++ b/src/Foundatio.AWS/Extensions/Extensions.cs
@@ -19,6 +19,10 @@ internal static FileSpec ToFileInfo(this S3Object blob)
             if (blob == null)
                 return null;
 
+            // Skip directories
+            if (blob.Key is not null&& blob.Size is 0 && blob.Key.EndsWith("/"))
+                return null;
+
             return new FileSpec
             {
                 Path = blob.Key,
@@ -30,7 +34,14 @@ internal static FileSpec ToFileInfo(this S3Object blob)
 
         internal static IEnumerable<S3Object> MatchesPattern(this IEnumerable<S3Object> blobs, Regex patternRegex)
         {
-            return blobs.Where(blob => patternRegex == null || patternRegex.IsMatch(blob.ToFileInfo().Path));
+            return blobs.Where(blob =>
+            {
+                var info = blob.ToFileInfo();
+                if (info?.Path is null)
+                    return false;
+
+                return patternRegex == null || patternRegex.IsMatch(info.Path);
+            });
         }
     }
 }
diff --git a/src/Foundatio.AWS/Metrics/CloudWatchMetricsClient.cs b/src/Foundatio.AWS/Metrics/CloudWatchMetricsClient.cs
index 42386d8..0d41190 100644
--- a/src/Foundatio.AWS/Metrics/CloudWatchMetricsClient.cs
+++ b/src/Foundatio.AWS/Metrics/CloudWatchMetricsClient.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using Amazon;
 using Amazon.CloudWatch;
 using Amazon.CloudWatch.Model;
 using Amazon.Runtime;
diff --git a/src/Foundatio.AWS/Queues/SQSQueueEntry.cs b/src/Foundatio.AWS/Queues/SQSQueueEntry.cs
index a464fac..2e7704b 100644
--- a/src/Foundatio.AWS/Queues/SQSQueueEntry.cs
+++ b/src/Foundatio.AWS/Queues/SQSQueueEntry.cs
@@ -1,4 +1,3 @@
-using System;
 using System.Linq;
 using Amazon.SQS.Model;
 
diff --git a/tests/Foundatio.AWS.Tests/Storage/S3FileStorageTests.cs b/tests/Foundatio.AWS.Tests/Storage/S3FileStorageTests.cs
index ead3e68..69b5a3d 100644
--- a/tests/Foundatio.AWS.Tests/Storage/S3FileStorageTests.cs
+++ b/tests/Foundatio.AWS.Tests/Storage/S3FileStorageTests.cs
@@ -1,8 +1,6 @@
 using System;
 using System.Threading.Tasks;
-using Amazon;
-using Amazon.Runtime;
-using Amazon.S3;
+using Amazon.S3.Model;
 using Foundatio.Storage;
 using Foundatio.Tests.Storage;
 using Xunit;
@@ -12,12 +10,14 @@ namespace Foundatio.AWS.Tests.Storage
 {
     public class S3FileStorageTests : FileStorageTestsBase
     {
+        private const string BUCKET_NAME = "foundatio-ci";
+
         public S3FileStorageTests(ITestOutputHelper output) : base(output) { }
 
         protected override IFileStorage GetStorage()
         {
             return new S3FileStorage(
-                o => o.ConnectionString($"serviceurl=http://localhost:4566;bucket=foundatio-ci;AccessKey=xxx;SecretKey=xxx")
+                o => o.ConnectionString($"serviceurl=http://localhost:4566;bucket={BUCKET_NAME};AccessKey=xxx;SecretKey=xxx")
                     .LoggerFactory(Log));
         }
 
@@ -129,17 +129,51 @@ public override Task WillRespectStreamOffsetAsync()
             return base.WillRespectStreamOffsetAsync();
         }
 
-        protected override async Task ResetAsync(IFileStorage storage)
+        [Fact]
+        public virtual async Task WillNotReturnDirectoryInGetPagedFileListAsync()
         {
-            var client = new AmazonS3Client(
-                new BasicAWSCredentials("xxx", "xxx"),
-                new AmazonS3Config
+            var storage = GetStorage();
+            if (storage == null)
+                return;
+
+            await ResetAsync(storage);
+
+            using (storage)
+            {
+                var result = await storage.GetPagedFileListAsync();
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+                Assert.False(await result.NextPageAsync());
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+
+                // To create an empty folder (or what appears as a folder) in an Amazon S3 bucket using the AWS SDK for .NET,
+                // you typically create an object with a key that ends with a trailing slash ('/') because S3 doesn't
+                // actually have a concept of folders, but it mimics the behavior of folders using object keys.
+                var client = storage is S3FileStorage s3Storage ? s3Storage.Client : null;
+                Assert.NotNull(client);
+
+                await client.PutObjectAsync(new PutObjectRequest
                 {
-                    RegionEndpoint = RegionEndpoint.USEast1,
-                    ServiceURL = "http://localhost:4566",
-                    ForcePathStyle = true
+                    BucketName = BUCKET_NAME,
+                    Key = "EmptyFolder/",
+                    ContentBody = String.Empty
                 });
-            await client.PutBucketAsync("foundatio-ci");
+
+                result = await storage.GetPagedFileListAsync();
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+                Assert.False(await result.NextPageAsync());
+                Assert.False(result.HasMore);
+                Assert.Empty(result.Files);
+            }
+        }
+
+        protected override async Task ResetAsync(IFileStorage storage)
+        {
+            var client = storage is S3FileStorage s3Storage ? s3Storage.Client : null;
+            Assert.NotNull(client);
+            await client.PutBucketAsync(BUCKET_NAME);
 
             await base.ResetAsync(storage);
         }