Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace with async lock #294

Merged
merged 4 commits into from
Sep 24, 2023
Merged

Replace with async lock #294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4373fb3
Merge pull request #1 from FoundatioFx/master
garcipat Sep 23, 2023
bb44c02
chore: use AsyncLock instead of regular lock since methods are async
garcipat Sep 23, 2023
0eace55
Merge branch 'master' into replace-with-async-lock
ejsmith Sep 24, 2023
b6083a3
Fix merge
ejsmith Sep 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/Foundatio/Storage/FolderFileStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Foundatio.AsyncEx;
using Foundatio.Extensions;
using Foundatio.Serializer;
using Foundatio.Utility;
Expand All @@ -12,7 +13,7 @@

namespace Foundatio.Storage {
public class FolderFileStorage : IFileStorage {
private readonly object _lockObject = new();
private readonly AsyncLock _lock = new();
private readonly ISerializer _serializer;
protected readonly ILogger _logger;

Expand Down Expand Up @@ -150,7 +151,7 @@ private Stream CreateFileStream(string filePath) {
return File.Create(filePath);
}

public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken cancellationToken = default) {
public async Task<bool> RenameFileAsync(string path, string newPath, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));
if (String.IsNullOrEmpty(newPath))
Expand All @@ -161,7 +162,7 @@ public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken
_logger.LogInformation("Renaming {Path} to {NewPath}", normalizedPath, normalizedNewPath);

try {
lock (_lockObject) {
using (await _lock.LockAsync().AnyContext()) {
string directory = Path.GetDirectoryName(normalizedNewPath);
if (directory != null) {
_logger.LogInformation("Creating {Directory} directory", directory);
Expand All @@ -182,13 +183,13 @@ public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken
}
} catch (Exception ex) {
_logger.LogError(ex, "Error renaming {Path} to {NewPath}", normalizedPath, normalizedNewPath);
return Task.FromResult(false);
return false;
}

return Task.FromResult(true);
return true;
}

public Task<bool> CopyFileAsync(string path, string targetPath, CancellationToken cancellationToken = default) {
public async Task<bool> CopyFileAsync(string path, string targetPath, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));
if (String.IsNullOrEmpty(targetPath))
Expand All @@ -199,7 +200,7 @@ public Task<bool> CopyFileAsync(string path, string targetPath, CancellationToke
_logger.LogInformation("Copying {Path} to {TargetPath}", normalizedPath, normalizedTargetPath);

try {
lock (_lockObject) {
using (await _lock.LockAsync().AnyContext()) {
string directory = Path.GetDirectoryName(normalizedTargetPath);
if (directory != null) {
_logger.LogInformation("Creating {Directory} directory", directory);
Expand All @@ -210,10 +211,10 @@ public Task<bool> CopyFileAsync(string path, string targetPath, CancellationToke
}
} catch (Exception ex) {
_logger.LogError(ex, "Error copying {Path} to {TargetPath}: {Message}", normalizedPath, normalizedTargetPath, ex.Message);
return Task.FromResult(false);
return false;
}

return Task.FromResult(true);
return true;
}

public Task<bool> DeleteFileAsync(string path, CancellationToken cancellationToken = default) {
Expand Down
95 changes: 50 additions & 45 deletions src/Foundatio/Storage/InMemoryFileStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,21 @@
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Foundatio.Utility;
using Foundatio.AsyncEx;
using Foundatio.Extensions;
using Foundatio.Serializer;
using Foundatio.Utility;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;

namespace Foundatio.Storage {
public class InMemoryFileStorage : IFileStorage {
private readonly Dictionary<string, Tuple<FileSpec, byte[]>> _storage = new(StringComparer.OrdinalIgnoreCase);
private readonly object _lock = new();
private readonly AsyncLock _lock = new();
private readonly ISerializer _serializer;
protected readonly ILogger _logger;

public InMemoryFileStorage() : this(o => o) {}
public InMemoryFileStorage() : this(o => o) { }

public InMemoryFileStorage(InMemoryFileStorageOptions options) {
if (options == null)
Expand All @@ -30,7 +31,7 @@ public InMemoryFileStorage(InMemoryFileStorageOptions options) {
_logger = options.LoggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance;
}

public InMemoryFileStorage(Builder<InMemoryFileStorageOptionsBuilder, InMemoryFileStorageOptions> config)
public InMemoryFileStorage(Builder<InMemoryFileStorageOptionsBuilder, InMemoryFileStorageOptions> config)
: this(config(new InMemoryFileStorageOptionsBuilder()).Build()) { }

public long MaxFileSize { get; set; }
Expand All @@ -40,20 +41,20 @@ public InMemoryFileStorage(Builder<InMemoryFileStorageOptionsBuilder, InMemoryFi
public Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken = default) =>
GetFileStreamAsync(path, StreamMode.Read, cancellationToken);

public Task<Stream> GetFileStreamAsync(string path, StreamMode streamMode, CancellationToken cancellationToken = default) {
public async Task<Stream> GetFileStreamAsync(string path, StreamMode streamMode, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));

string normalizedPath = path.NormalizePath();
_logger.LogTrace("Getting file stream for {Path}", normalizedPath);

lock (_lock) {
using (await _lock.LockAsync().AnyContext()) {
if (!_storage.ContainsKey(normalizedPath)) {
_logger.LogError("Unable to get file stream for {Path}: File Not Found", normalizedPath);
return Task.FromResult<Stream>(null);
return null;
}

return Task.FromResult<Stream>(new MemoryStream(_storage[normalizedPath].Item2));
return new MemoryStream(_storage[normalizedPath].Item2);
}
}

Expand All @@ -71,13 +72,16 @@ public async Task<FileSpec> GetFileInfoAsync(string path) {
return null;
}

public Task<bool> ExistsAsync(string path) {
public async Task<bool> ExistsAsync(string path) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));

string normalizedPath = path.NormalizePath();
_logger.LogTrace("Checking if {Path} exists", normalizedPath);
return Task.FromResult(_storage.ContainsKey(normalizedPath));

using (await _lock.LockAsync().AnyContext()) {
return _storage.ContainsKey(normalizedPath);
}
}

private static byte[] ReadBytes(Stream input) {
Expand All @@ -86,20 +90,20 @@ private static byte[] ReadBytes(Stream input) {
return ms.ToArray();
}

public Task<bool> SaveFileAsync(string path, Stream stream, CancellationToken cancellationToken = default) {
public async Task<bool> SaveFileAsync(string path, Stream stream, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));
if (stream == null)
throw new ArgumentNullException(nameof(stream));

string normalizedPath = path.NormalizePath();
_logger.LogTrace("Saving {Path}", normalizedPath);

var contents = ReadBytes(stream);
if (contents.Length > MaxFileSize)
throw new ArgumentException($"File size {contents.Length.ToFileSizeDisplay()} exceeds the maximum size of {MaxFileSize.ToFileSizeDisplay()}.");

lock (_lock) {
using (await _lock.LockAsync().AnyContext()) {
_storage[normalizedPath] = Tuple.Create(new FileSpec {
Created = SystemClock.UtcNow,
Modified = SystemClock.UtcNow,
Expand All @@ -111,10 +115,10 @@ public Task<bool> SaveFileAsync(string path, Stream stream, CancellationToken ca
_storage.Remove(_storage.OrderByDescending(kvp => kvp.Value.Item1.Created).First().Key);
}

return Task.FromResult(true);
return true;
}

public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken cancellationToken = default) {
public async Task<bool> RenameFileAsync(string path, string newPath, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));
if (String.IsNullOrEmpty(newPath))
Expand All @@ -123,11 +127,11 @@ public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken
string normalizedPath = path.NormalizePath();
string normalizedNewPath = newPath.NormalizePath();
_logger.LogInformation("Renaming {Path} to {NewPath}", normalizedPath, normalizedNewPath);
lock (_lock) {

using (await _lock.LockAsync().AnyContext()) {
if (!_storage.ContainsKey(normalizedPath)) {
_logger.LogDebug("Error renaming {Path} to {NewPath}: File not found", normalizedPath, normalizedNewPath);
return Task.FromResult(false);
return false;
}

_storage[normalizedNewPath] = _storage[normalizedPath];
Expand All @@ -136,10 +140,10 @@ public Task<bool> RenameFileAsync(string path, string newPath, CancellationToken
_storage.Remove(normalizedPath);
}

return Task.FromResult(true);
return true;
}

public Task<bool> CopyFileAsync(string path, string targetPath, CancellationToken cancellationToken = default) {
public async Task<bool> CopyFileAsync(string path, string targetPath, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));
if (String.IsNullOrEmpty(targetPath))
Expand All @@ -148,72 +152,73 @@ public Task<bool> CopyFileAsync(string path, string targetPath, CancellationToke
string normalizedPath = path.NormalizePath();
string normalizedTargetPath = targetPath.NormalizePath();
_logger.LogInformation("Copying {Path} to {TargetPath}", normalizedPath, normalizedTargetPath);
lock (_lock) {

using (await _lock.LockAsync().AnyContext()) {
if (!_storage.ContainsKey(normalizedPath)) {
_logger.LogDebug("Error copying {Path} to {TargetPath}: File not found", normalizedPath, normalizedTargetPath);
return Task.FromResult(false);
return false;
}

_storage[normalizedTargetPath] = _storage[normalizedPath];
_storage[normalizedTargetPath].Item1.Path = normalizedTargetPath;
_storage[normalizedTargetPath].Item1.Modified = SystemClock.UtcNow;
}

return Task.FromResult(true);
return true;
}

public Task<bool> DeleteFileAsync(string path, CancellationToken cancellationToken = default) {
public async Task<bool> DeleteFileAsync(string path, CancellationToken cancellationToken = default) {
if (String.IsNullOrEmpty(path))
throw new ArgumentNullException(nameof(path));

string normalizedPath = path.NormalizePath();
_logger.LogTrace("Deleting {Path}", normalizedPath);
lock (_lock) {

using (await _lock.LockAsync().AnyContext()) {
if (!_storage.ContainsKey(normalizedPath)) {
_logger.LogError("Unable to delete {Path}: File not found", normalizedPath);
return Task.FromResult(false);
return false;
}

_storage.Remove(normalizedPath);
}

return Task.FromResult(true);
return true;
}

public Task<int> DeleteFilesAsync(string searchPattern = null, CancellationToken cancellation = default) {
public async Task<int> DeleteFilesAsync(string searchPattern = null, CancellationToken cancellation = default) {
if (String.IsNullOrEmpty(searchPattern) || searchPattern == "*") {
lock(_lock)
using (await _lock.LockAsync().AnyContext()) {
_storage.Clear();
}

return Task.FromResult(0);
return 0;
}

searchPattern = searchPattern.NormalizePath();
int count = 0;

if (searchPattern[searchPattern.Length - 1] == Path.DirectorySeparatorChar)
if (searchPattern[searchPattern.Length - 1] == Path.DirectorySeparatorChar)
searchPattern = $"{searchPattern}*";
else if (!searchPattern.EndsWith(Path.DirectorySeparatorChar + "*") && !Path.HasExtension(searchPattern))
searchPattern = Path.Combine(searchPattern, "*");

var regex = new Regex($"^{Regex.Escape(searchPattern).Replace("\\*", ".*?")}$");
lock (_lock) {

using (await _lock.LockAsync().AnyContext()) {
var keys = _storage.Keys.Where(k => regex.IsMatch(k)).Select(k => _storage[k].Item1).ToList();

_logger.LogInformation("Deleting {FileCount} files matching {SearchPattern} (Regex={SearchPatternRegex})", keys.Count, searchPattern, regex);
foreach (var key in keys) {
_logger.LogTrace("Deleting {Path}", key.Path);
_storage.Remove(key.Path);
count++;
}

_logger.LogTrace("Finished deleting {FileCount} files matching {SearchPattern}", count, searchPattern);
}

return Task.FromResult(count);
return count;
}

public async Task<PagedFileListResult> GetPagedFileListAsync(int pageSize = 100, string searchPattern = null, CancellationToken cancellationToken = default) {
Expand All @@ -225,21 +230,21 @@ public async Task<PagedFileListResult> GetPagedFileListAsync(int pageSize = 100,

searchPattern = searchPattern.NormalizePath();

var result = new PagedFileListResult(s => Task.FromResult(GetFiles(searchPattern, 1, pageSize)));
var result = new PagedFileListResult(async s => await GetFilesAsync(searchPattern, 1, pageSize, cancellationToken));
await result.NextPageAsync().AnyContext();
return result;
}

private NextPageResult GetFiles(string searchPattern, int page, int pageSize) {
private async Task<NextPageResult> GetFilesAsync(string searchPattern, int page, int pageSize, CancellationToken cancellationToken = default) {
var list = new List<FileSpec>();
int pagingLimit = pageSize;
int skip = (page - 1) * pagingLimit;
if (pagingLimit < Int32.MaxValue)
pagingLimit++;

var regex = new Regex($"^{Regex.Escape(searchPattern).Replace("\\*", ".*?")}$");

lock (_lock) {
using (await _lock.LockAsync().AnyContext()) {
_logger.LogTrace(s => s.Property("Limit", pagingLimit).Property("Skip", skip), "Getting file list matching {SearchPattern}...", regex);
list.AddRange(_storage.Keys.Where(k => regex.IsMatch(k)).Select(k => _storage[k].Item1.DeepClone()).Skip(skip).Take(pagingLimit).ToList());
}
Expand All @@ -251,10 +256,10 @@ private NextPageResult GetFiles(string searchPattern, int page, int pageSize) {
}

return new NextPageResult {
Success = true,
HasMore = hasMore,
Success = true,
HasMore = hasMore,
Files = list,
NextPageFunc = hasMore ? _ => Task.FromResult(GetFiles(searchPattern, page + 1, pageSize)) : null
NextPageFunc = hasMore ? async _ => await GetFilesAsync(searchPattern, page + 1, pageSize, cancellationToken) : null
};
}

Expand Down