From 67241d6d9cfeb4b7c35b1ef451f48c517be8898b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 6 Dec 2021 18:02:26 -0600 Subject: [PATCH] Don't use deep clone in scoped file storage --- Foundatio.All.sln | 47 +++++++++- .../Storage/FileStorageTestsBase.cs | 87 ++++++++++--------- src/Foundatio/Storage/InMemoryFileStorage.cs | 6 +- src/Foundatio/Storage/ScopedFileStorage.cs | 4 +- 4 files changed, 95 insertions(+), 49 deletions(-) diff --git a/Foundatio.All.sln b/Foundatio.All.sln index 2ec315a8a..82457b276 100644 --- a/Foundatio.All.sln +++ b/Foundatio.All.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.28606.126 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31912.275 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{70515E66-DAF8-4D18-8F8F-8A2934171AA9}" EndProject @@ -85,6 +85,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Repositories.Test EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Storage.SshNet.Tests", "..\Foundatio.Storage.SshNet\tests\Foundatio.Storage.SshNet.Tests\Foundatio.Storage.SshNet.Tests.csproj", "{AD5EAA6E-E914-4F6E-80F0-D09FC0D85E21}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Renci.SshNet", "..\..\Other\SSH.NET\src\Renci.SshNet\Renci.SshNet.csproj", "{7E024AD7-29CD-4E65-BD3E-F0716161D78E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Rebex", "..\Foundatio.Rebex\src\Foundatio.Rebex\Foundatio.Rebex.csproj", "{29968DA3-F74E-4759-A0A3-80E12698374B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Foundatio.Rebex.Tests", "..\Foundatio.Rebex\tests\Foundatio.Rebex.Tests\Foundatio.Rebex.Tests.csproj", "{F66E5836-E830-4279-9F4C-E3FEA923A992}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -503,6 +509,42 @@ Global {AD5EAA6E-E914-4F6E-80F0-D09FC0D85E21}.Release|x64.Build.0 = Release|Any CPU {AD5EAA6E-E914-4F6E-80F0-D09FC0D85E21}.Release|x86.ActiveCfg = Release|Any CPU {AD5EAA6E-E914-4F6E-80F0-D09FC0D85E21}.Release|x86.Build.0 = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|x64.Build.0 = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Debug|x86.Build.0 = Debug|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|Any CPU.Build.0 = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|x64.ActiveCfg = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|x64.Build.0 = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|x86.ActiveCfg = Release|Any CPU + {7E024AD7-29CD-4E65-BD3E-F0716161D78E}.Release|x86.Build.0 = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|x64.ActiveCfg = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|x64.Build.0 = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|x86.ActiveCfg = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Debug|x86.Build.0 = Debug|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|Any CPU.Build.0 = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|x64.ActiveCfg = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|x64.Build.0 = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|x86.ActiveCfg = Release|Any CPU + {29968DA3-F74E-4759-A0A3-80E12698374B}.Release|x86.Build.0 = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|x64.ActiveCfg = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|x64.Build.0 = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|x86.ActiveCfg = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Debug|x86.Build.0 = Debug|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|Any CPU.Build.0 = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|x64.ActiveCfg = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|x64.Build.0 = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|x86.ActiveCfg = Release|Any CPU + {F66E5836-E830-4279-9F4C-E3FEA923A992}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -522,6 +564,7 @@ Global {EAF2535F-F943-499C-9260-0ADA6F0523C2} = {70515E66-DAF8-4D18-8F8F-8A2934171AA9} {488C58C3-6510-4A77-9EFC-30041E8B28A1} = {70515E66-DAF8-4D18-8F8F-8A2934171AA9} {AD5EAA6E-E914-4F6E-80F0-D09FC0D85E21} = {70515E66-DAF8-4D18-8F8F-8A2934171AA9} + {F66E5836-E830-4279-9F4C-E3FEA923A992} = {70515E66-DAF8-4D18-8F8F-8A2934171AA9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BD75F5C6-658B-40DF-A8F6-91C3EB76DAAA} diff --git a/src/Foundatio.TestHarness/Storage/FileStorageTestsBase.cs b/src/Foundatio.TestHarness/Storage/FileStorageTestsBase.cs index 1951efbcd..c45b77e5e 100644 --- a/src/Foundatio.TestHarness/Storage/FileStorageTestsBase.cs +++ b/src/Foundatio.TestHarness/Storage/FileStorageTestsBase.cs @@ -23,24 +23,24 @@ protected virtual IFileStorage GetStorage() { } public virtual async Task CanGetEmptyFileListOnMissingDirectoryAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { Assert.Empty(await storage.GetFileListAsync(Guid.NewGuid() + "\\*")); } } public virtual async Task CanGetFileListForSingleFolderAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"archived\archived.txt", "archived"); await storage.SaveFileAsync(@"q\new.txt", "new"); @@ -59,12 +59,12 @@ public virtual async Task CanGetFileListForSingleFolderAsync() { } public virtual async Task CanGetFileListForSingleFileAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"archived\archived.txt", "archived"); await storage.SaveFileAsync(@"archived\archived.csv", "archived"); @@ -76,12 +76,12 @@ public virtual async Task CanGetFileListForSingleFileAsync() { } public virtual async Task CanGetPagedFileListForSingleFolderAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { var result = await storage.GetPagedFileListAsync(1); Assert.False(result.HasMore); @@ -121,12 +121,12 @@ public virtual async Task CanGetPagedFileListForSingleFolderAsync() { } public virtual async Task CanGetFileInfoAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { var fileInfo = await storage.GetFileInfoAsync(Guid.NewGuid().ToString()); Assert.Null(fileInfo); @@ -158,12 +158,12 @@ public virtual async Task CanGetFileInfoAsync() { } public virtual async Task CanGetNonExistentFileInfoAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await Assert.ThrowsAnyAsync(() => storage.GetFileInfoAsync(null)); Assert.Null(await storage.GetFileInfoAsync(Guid.NewGuid().ToString())); @@ -171,12 +171,12 @@ public virtual async Task CanGetNonExistentFileInfoAsync() { } public virtual async Task CanManageFilesAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync("test.txt", "test"); var file = (await storage.GetFileListAsync()).Single(); @@ -192,12 +192,12 @@ public virtual async Task CanManageFilesAsync() { } public virtual async Task CanRenameFilesAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { Assert.True(await storage.SaveFileAsync("test.txt", "test")); Assert.True(await storage.RenameFileAsync("test.txt", @"archive\new.txt")); @@ -226,12 +226,12 @@ protected virtual string GetTestFilePath() { } public virtual async Task CanSaveFilesAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + string readmeFile = GetTestFilePath(); using (storage) { Assert.False(await storage.ExistsAsync("Foundatio.Tests.csproj")); @@ -252,12 +252,12 @@ public virtual async Task CanSaveFilesAsync() { } public virtual async Task CanDeleteEntireFolderAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"x\hello.txt", "hello"); await storage.SaveFileAsync(@"x\nested\world.csv", "nested world"); @@ -269,12 +269,12 @@ public virtual async Task CanDeleteEntireFolderAsync() { } public virtual async Task CanDeleteEntireFolderWithWildcardAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"x\hello.txt", "hello"); await storage.SaveFileAsync(@"x\nested\world.csv", "nested world"); @@ -290,12 +290,12 @@ public virtual async Task CanDeleteEntireFolderWithWildcardAsync() { } public virtual async Task CanDeleteFolderWithMultiFolderWildcardsAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { const int filesPerMonth = 5; for (int year = 2020; year <= 2021; year++) { @@ -305,10 +305,14 @@ public virtual async Task CanDeleteFolderWithMultiFolderWildcardsAsync() { } } + _logger.LogInformation(@"List by pattern: archive\*"); Assert.Equal(2 * 12 * filesPerMonth, (await storage.GetFileListAsync(@"archive\*")).Count); + _logger.LogInformation(@"List by pattern: archive\*month-01*"); Assert.Equal(2 * filesPerMonth, (await storage.GetFileListAsync(@"archive\*month-01*")).Count); + _logger.LogInformation(@"List by pattern: archive\year-2020\*month-01*"); Assert.Equal(filesPerMonth, (await storage.GetFileListAsync(@"archive\year-2020\*month-01*")).Count); + _logger.LogInformation(@"Delete by pattern: archive\*month-01*"); await storage.DeleteFilesAsync(@"archive\*month-01*"); Assert.Equal(2 * 11 * filesPerMonth, (await storage.GetFileListAsync()).Count); @@ -316,12 +320,12 @@ public virtual async Task CanDeleteFolderWithMultiFolderWildcardsAsync() { } public virtual async Task CanDeleteSpecificFilesAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"x\hello.txt", "hello"); await storage.SaveFileAsync(@"x\nested\world.csv", "nested world"); @@ -342,12 +346,12 @@ public virtual async Task CanDeleteSpecificFilesAsync() { } public virtual async Task CanDeleteNestedFolderAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"x\hello.txt", "hello"); await storage.SaveFileAsync(@"x\nested\world.csv", "nested world"); @@ -368,12 +372,12 @@ public virtual async Task CanDeleteNestedFolderAsync() { } public virtual async Task CanDeleteSpecificFilesInNestedFolderAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { await storage.SaveFileAsync(@"x\hello.txt", "hello"); await storage.SaveFileAsync(@"x\world.csv", "world"); @@ -398,12 +402,12 @@ public virtual async Task CanDeleteSpecificFilesInNestedFolderAsync() { } public virtual async Task CanRoundTripSeekableStreamAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { const string path = "user.xml"; var element = XElement.Parse("Blake"); @@ -426,12 +430,12 @@ public virtual async Task CanRoundTripSeekableStreamAsync() { } public virtual async Task WillRespectStreamOffsetAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { string path = "blake.txt"; using (var memoryStream = new MemoryStream()) { @@ -452,24 +456,23 @@ public virtual async Task WillRespectStreamOffsetAsync() { } } - protected virtual async Task ResetAsync() { - var storage = GetStorage(); + protected virtual async Task ResetAsync(IFileStorage storage) { if (storage == null) return; - using (storage) { - await storage.DeleteFilesAsync(); - Assert.Empty(await storage.GetFileListAsync()); - } + _logger.LogInformation("Deleting all files..."); + await storage.DeleteFilesAsync(); + _logger.LogInformation("Asserting empty files..."); + Assert.Empty(await storage.GetFileListAsync(limit: 10000)); } public virtual async Task CanConcurrentlyManageFilesAsync() { - await ResetAsync(); - var storage = GetStorage(); if (storage == null) return; + await ResetAsync(storage); + using (storage) { const string queueFolder = "q"; var queueItems = new BlockingCollection(); diff --git a/src/Foundatio/Storage/InMemoryFileStorage.cs b/src/Foundatio/Storage/InMemoryFileStorage.cs index 837d66ed1..21306f9e4 100644 --- a/src/Foundatio/Storage/InMemoryFileStorage.cs +++ b/src/Foundatio/Storage/InMemoryFileStorage.cs @@ -51,7 +51,7 @@ public async Task GetFileInfoAsync(string path) { throw new ArgumentNullException(nameof(path)); path = path.NormalizePath(); - return await ExistsAsync(path).AnyContext() ? _storage[path].Item1 : null; + return await ExistsAsync(path).AnyContext() ? _storage[path].Item1.DeepClone() : null; } public Task ExistsAsync(string path) { @@ -204,7 +204,7 @@ private NextPageResult GetFiles(string searchPattern, int page, int pageSize) { var regex = new Regex("^" + Regex.Escape(searchPattern).Replace("\\*", ".*?") + "$"); lock (_lock) - list.AddRange(_storage.Keys.Where(k => regex.IsMatch(k)).Select(k => _storage[k].Item1).Skip(skip).Take(pagingLimit).ToList()); + list.AddRange(_storage.Keys.Where(k => regex.IsMatch(k)).Select(k => _storage[k].Item1.DeepClone()).Skip(skip).Take(pagingLimit).ToList()); bool hasMore = false; if (list.Count == pagingLimit) { @@ -216,7 +216,7 @@ private NextPageResult GetFiles(string searchPattern, int page, int pageSize) { Success = true, HasMore = hasMore, Files = list, - NextPageFunc = hasMore ? s => Task.FromResult(GetFiles(searchPattern, page + 1, pageSize)) : (Func>)null + NextPageFunc = hasMore ? s => Task.FromResult(GetFiles(searchPattern, page + 1, pageSize)) : null }; } diff --git a/src/Foundatio/Storage/ScopedFileStorage.cs b/src/Foundatio/Storage/ScopedFileStorage.cs index 93959b430..1428ca765 100644 --- a/src/Foundatio/Storage/ScopedFileStorage.cs +++ b/src/Foundatio/Storage/ScopedFileStorage.cs @@ -91,7 +91,7 @@ public async Task GetPagedFileListAsync(int pageSize = 100, searchPattern = !String.IsNullOrEmpty(searchPattern) ? String.Concat(_pathPrefix, searchPattern) : String.Concat(_pathPrefix, "*"); var unscopedResult = await UnscopedStorage.GetPagedFileListAsync(pageSize, searchPattern, cancellationToken).AnyContext(); - unscopedResult = unscopedResult.DeepClone(); + foreach (var file in unscopedResult.Files) file.Path = file.Path.Substring(_pathPrefix.Length); @@ -100,7 +100,7 @@ public async Task GetPagedFileListAsync(int pageSize = 100, private async Task NextPage(PagedFileListResult result) { var success = await result.NextPageAsync().AnyContext(); - result = result.DeepClone(); + foreach (var file in result.Files) file.Path = file.Path.Substring(_pathPrefix.Length);