diff --git a/FacwareBase.API/Controllers/Document/FileController.cs b/FacwareBase.API/Controllers/Document/FileController.cs index 012a8c0..67cc125 100644 --- a/FacwareBase.API/Controllers/Document/FileController.cs +++ b/FacwareBase.API/Controllers/Document/FileController.cs @@ -43,7 +43,7 @@ public async Task> AddFiles(string bucketName, str return BadRequest("The request doesn't contain any files to be uploaded."); } - var response = await _filesRepository.UploadFiles(bucketName, formFiles, key); + var response = await _filesRepository.UploadFiles(bucketName, key, formFiles[0]); if (response == null) { @@ -77,13 +77,16 @@ public async Task>> ListFiles(string [Route("{bucketName}/download/{fileName}")] public async Task DownloadFile(string bucketName, string fileName) { - var temporalPath = _fileStorageOptions.TemporalStorage; + //var temporalPath = _fileStorageOptions.TemporalStorage; - await _filesRepository.DownloadFile(bucketName, fileName, temporalPath); + // await _filesRepository.DownloadFile(bucketName, fileName, temporalPath); - var memoryFile = _fileManagement.ReadFileAsync(temporalPath, fileName).Result; + // var memoryFile = _fileManagement.ReadFileAsync(temporalPath, fileName).Result; - _fileManagement.RemoveFile(temporalPath, fileName); + // _fileManagement.RemoveFile(temporalPath, fileName); + + var key = "test123/"; + var memoryFile = await _filesRepository.DownloadMemoryStreamAsync(bucketName, key + fileName); var mimeType = _fileManagement.GetMimeType(fileName); diff --git a/FacwareBase.API/Properties/launchSettings.json b/FacwareBase.API/Properties/launchSettings.json index 2f5c4fc..5d7e29f 100644 --- a/FacwareBase.API/Properties/launchSettings.json +++ b/FacwareBase.API/Properties/launchSettings.json @@ -14,7 +14,7 @@ "launchBrowser": true, "launchUrl": "weatherforecast", "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" + "ASPNETCORE_ENVIRONMENT": "Local" } }, "FacwareBase.API": { diff --git a/FacwareBase.API/Services/Amazon/S3/Core/File/AddFileResponse.cs b/FacwareBase.API/Services/Amazon/S3/Core/File/AddFileResponse.cs index da882ca..da42c32 100644 --- a/FacwareBase.API/Services/Amazon/S3/Core/File/AddFileResponse.cs +++ b/FacwareBase.API/Services/Amazon/S3/Core/File/AddFileResponse.cs @@ -2,8 +2,29 @@ namespace FacwareBase.API.Services.Amazon.S3.Core.File { + /// + /// New file response + /// public class AddFileResponse { + /// + /// Uri + /// public IList PreSignedUrl { get; set; } } + + /// + /// file data + /// + public class AddSingleFileResponse + { + /// + /// url pre signed + /// + public string PreSignedUrl { get; set; } + /// + /// s3 key + /// + public string Key { get; set; } + } } \ No newline at end of file diff --git a/FacwareBase.API/Services/Amazon/S3/Core/File/AddItemRequest.cs b/FacwareBase.API/Services/Amazon/S3/Core/File/AddItemRequest.cs index 2f3b953..a8cf594 100644 --- a/FacwareBase.API/Services/Amazon/S3/Core/File/AddItemRequest.cs +++ b/FacwareBase.API/Services/Amazon/S3/Core/File/AddItemRequest.cs @@ -1,7 +1,9 @@ namespace FacwareBase.API.Services.Amazon.S3.Core.File { - public class AddItemRequest - { - - } + /// + /// New item request + /// + public class AddItemRequest + { + } } \ No newline at end of file diff --git a/FacwareBase.API/Services/Amazon/S3/Core/File/DeleteFileResponse.cs b/FacwareBase.API/Services/Amazon/S3/Core/File/DeleteFileResponse.cs index 4a8533e..0e82af8 100644 --- a/FacwareBase.API/Services/Amazon/S3/Core/File/DeleteFileResponse.cs +++ b/FacwareBase.API/Services/Amazon/S3/Core/File/DeleteFileResponse.cs @@ -1,7 +1,13 @@ namespace FacwareBase.API.Services.Amazon.S3.Core.File { + /// + /// Deleted objects summary + /// public class DeleteFileResponse { + /// + /// delete object number + /// public int NumberOfDeletedObjects { get; set; } } } \ No newline at end of file diff --git a/FacwareBase.API/Services/Amazon/S3/Core/File/GetJsonObjectResponse.cs b/FacwareBase.API/Services/Amazon/S3/Core/File/GetJsonObjectResponse.cs index 645f689..7bd84db 100644 --- a/FacwareBase.API/Services/Amazon/S3/Core/File/GetJsonObjectResponse.cs +++ b/FacwareBase.API/Services/Amazon/S3/Core/File/GetJsonObjectResponse.cs @@ -4,8 +4,17 @@ namespace FacwareBase.API.Services.Amazon.S3.Core.File { public class GetJsonObjectResponse { + /// + /// Id + /// public Guid Id { get; set; } + /// + /// Last update + /// public DateTime TimeSent { get; set; } + /// + /// Json data + /// public string Data { get; set; } } } \ No newline at end of file diff --git a/FacwareBase.API/Services/Amazon/S3/Core/Interfaces/IFilesRepository.cs b/FacwareBase.API/Services/Amazon/S3/Core/Interfaces/IFilesRepository.cs index 0b612ee..1152295 100644 --- a/FacwareBase.API/Services/Amazon/S3/Core/Interfaces/IFilesRepository.cs +++ b/FacwareBase.API/Services/Amazon/S3/Core/Interfaces/IFilesRepository.cs @@ -1,52 +1,54 @@ using FacwareBase.API.Services.Amazon.S3.Core.File; using Microsoft.AspNetCore.Http; using System.Collections.Generic; +using System.IO; using System.Threading.Tasks; namespace FacwareBase.API.Services.Amazon.S3.Core.Interfaces { public interface IFilesRepository - { + { + /// + /// Add json object + /// + /// bucket name + /// object request + Task AddJsonObject(string bucketName, AddJsonObjectRequest request); + /// - /// Upload files + /// Copy a file or key to another location /// - /// bucket name - /// files - /// s3 key - /// AddFileResponse - Task UploadFiles(string bucketName, IList formFiles, string key = ""); + /// source bucket name + /// source key + /// target bucket name + /// traget key + /// target key + Task CopyKey(string sourceBucket, string sourceKey, string targetBucket, string targetKey); /// - /// List all key/files in a s3 bucket + /// Delete file from s3 /// /// bucket name - /// ListFilesResponse - Task> ListFiles(string bucketName); + /// s3 key /key + /// DeleteFileResponse + Task DeleteFile(string bucketName, string key); /// /// Download file from s3 to local temp folder /// /// bucket name - /// file name - /// temporal path to download - /// s3 key: path + /// key = path + file name + /// temporal path to download + file name /// task just to download file - Task DownloadFile(string bucketName, string fileName, string temporalPath, string key = ""); + Task DownloadFile(string bucketName, string key, string temporalPath); /// - /// Delete file from s3 - /// - /// bucket name - /// file name /key - /// DeleteFileResponse - Task DeleteFile(string bucketName, string fileName); - - /// - /// Add json object + /// Get file from s3 bucket /// /// bucket name - /// object request - Task AddJsonObject(string bucketName, AddJsonObjectRequest request); + /// s3 key - path + file name + /// Memory file object + Task DownloadMemoryStreamAsync(string bucketName, string key); /// /// Get json object @@ -55,5 +57,21 @@ public interface IFilesRepository /// file name /// GetJsonObjectResponse Task GetJsonObject(string bucketName, string fileName); + + /// + /// List all key/files in a s3 bucket + /// + /// bucket name + /// ListFilesResponse + Task> ListFiles(string bucketName); + + /// + /// Upload files + /// + /// bucket name + /// files + /// s3 key + /// AddFileResponse + Task UploadFiles(string bucketName, string key, params IFormFile[] formFiles); } } \ No newline at end of file diff --git a/FacwareBase.API/Services/Amazon/S3/Infraestructure/Repositories/FilesRepository.cs b/FacwareBase.API/Services/Amazon/S3/Infraestructure/Repositories/FilesRepository.cs index 38f0875..50cfd7d 100644 --- a/FacwareBase.API/Services/Amazon/S3/Infraestructure/Repositories/FilesRepository.cs +++ b/FacwareBase.API/Services/Amazon/S3/Infraestructure/Repositories/FilesRepository.cs @@ -27,42 +27,67 @@ public class FilesRepository : IFilesRepository public FilesRepository(IAmazonS3 s3Client, IOptions sessionAwsCredentialsOptions) { - _sessionAwsCredentialsOptions = sessionAwsCredentialsOptions.Value; - // TODO: we must improve this client initialization, this code commented is just to test in the local environment - // create a temporal session to test locally, use SSO temporal keys, update these keys in app settings - //SessionAWSCredentials tempCredentials = new SessionAWSCredentials(_sessionAwsCredentialsOptions.AwsAccessKeyId, - // _sessionAwsCredentialsOptions.AwsSecretAccessKey, - // _sessionAwsCredentialsOptions.Token); - //_s3Client = new AmazonS3Client(tempCredentials, RegionEndpoint.APSoutheast1); - - // IAM - //var credentials = new BasicAWSCredentials(_sessionAwsCredentialsOptions.AwsAccessKeyId, _sessionAwsCredentialsOptions.AwsSecretAccessKey); - //_s3Client = new AmazonS3Client(credentials, RegionEndpoint.USEast1); - - // deployed to development, staging or production - _s3Client = s3Client; + var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + + _sessionAwsCredentialsOptions = sessionAwsCredentialsOptions.Value; + if (env.Contains("Local")) + { + // TODO: we must improve this client initialization, this code commented is just to test in the local environment. create a temporal session to test locally, use SSO temporal keys, update these keys in app settings + SessionAWSCredentials tempCredentials = new SessionAWSCredentials(_sessionAwsCredentialsOptions.AwsAccessKeyId, + _sessionAwsCredentialsOptions.AwsSecretAccessKey, + _sessionAwsCredentialsOptions.Token); + _s3Client = new AmazonS3Client(tempCredentials, RegionEndpoint.APSoutheast1); + } + else + { + // TODO: example using IAM keys + // IAM + //var credentials = new BasicAWSCredentials(_sessionAwsCredentialsOptions.AwsAccessKeyId, _sessionAwsCredentialsOptions.AwsSecretAccessKey); + //_s3Client = new AmazonS3Client(RegionEndpoint.APSoutheast1); + + // TODO: using IAM role, deployed to development, staging or production + _s3Client = s3Client; + } } private static async Task GetTemporaryCredentialsAsync() { - using (var stsClient = new AmazonSecurityTokenServiceClient()) + using var stsClient = new AmazonSecurityTokenServiceClient(); + var getSessionTokenRequest = new GetSessionTokenRequest + { + DurationSeconds = 7200 // seconds + }; + + GetSessionTokenResponse sessionTokenResponse = + await stsClient.GetSessionTokenAsync(getSessionTokenRequest); + + Credentials credentials = sessionTokenResponse.Credentials; + + var sessionCredentials = + new SessionAWSCredentials(credentials.AccessKeyId, + credentials.SecretAccessKey, + credentials.SessionToken); + return sessionCredentials; + } + + /// + /// List all key/files in a s3 bucket + /// + /// bucket name + /// ListFilesResponse + public async Task> ListFiles(string bucketName) + { + var responses = await _s3Client.ListObjectsAsync(bucketName); + + return responses.S3Objects.Select(b => new ListFilesResponse { - var getSessionTokenRequest = new GetSessionTokenRequest - { - DurationSeconds = 7200 // seconds - }; - - GetSessionTokenResponse sessionTokenResponse = - await stsClient.GetSessionTokenAsync(getSessionTokenRequest); - - Credentials credentials = sessionTokenResponse.Credentials; - - var sessionCredentials = - new SessionAWSCredentials(credentials.AccessKeyId, - credentials.SecretAccessKey, - credentials.SessionToken); - return sessionCredentials; - } + BucketName = b.BucketName, + Key = b.Key, + Owner = b.Owner.DisplayName, + Size = b.Size, + LastModified = b.LastModified, + StorageClass = b.StorageClass + }); } /// @@ -72,20 +97,20 @@ private static async Task GetTemporaryCredentialsAsync() /// files /// s3 key /// AddFileResponse - public async Task UploadFiles(string bucketName, IList formFiles, string key) + public async Task UploadFiles(string bucketName, string key, params IFormFile[] formFiles) { var response = new List(); - var fullKey = key.Equals(string.Empty) ? key : $"{key}/"; - foreach (var file in formFiles) { var uploadRequest = new TransferUtilityUploadRequest { InputStream = file.OpenReadStream(), - Key = fullKey + file.FileName, + Key = key, BucketName = bucketName, - CannedACL = S3CannedACL.NoACL + CannedACL = S3CannedACL.PublicRead, + StorageClass = S3StorageClass.Standard, + ContentType = file.ContentType }; using (var fileTransferUtility = new TransferUtility(_s3Client)) @@ -96,7 +121,7 @@ public async Task UploadFiles(string bucketName, IList UploadFiles(string bucketName, IList - /// List all key/files in a s3 bucket + /// Add json object /// /// bucket name - /// ListFilesResponse - public async Task> ListFiles(string bucketName) + /// object request + public async Task AddJsonObject(string bucketName, AddJsonObjectRequest request) { - var responses = await _s3Client.ListObjectsAsync(bucketName); + var createdOnUtc = DateTime.UtcNow; + + var s3Key = $"{createdOnUtc:yyyy}/{createdOnUtc:MM}/{createdOnUtc:dd}/{request.Id}"; - return responses.S3Objects.Select(b => new ListFilesResponse + var putObjectRequest = new PutObjectRequest { - BucketName = b.BucketName, - Key = b.Key, - Owner = b.Owner.DisplayName, - Size = b.Size, - LastModified = b.LastModified, - StorageClass = b.StorageClass - }); + BucketName = bucketName, + Key = s3Key, + ContentBody = JsonConvert.SerializeObject(request) + }; + + await _s3Client.PutObjectAsync(putObjectRequest); } /// - /// Download file from s3 to local temp folder + /// Copy a file or key to another location /// - /// bucket name - /// file name - /// temporal path to download - /// s3 key: path - /// task just to download file - public async Task DownloadFile(string bucketName, string fileName, string temporalPath, string key = "") + /// source bucket name + /// source key + /// target bucket name + /// traget key + /// target key + public async Task CopyKey(string sourceBucket, string sourceKey, string targetBucket, string targetKey) { - var fullKey = key.Equals(string.Empty) ? $"{fileName}" : $"{key}/{fileName}"; - - GetObjectRequest request = new GetObjectRequest(); - request.BucketName = bucketName; - request.Key = fullKey; - // GetObjectResponse response = await _s3Client.GetObjectAsync(request); - - var downloadRequest = new TransferUtilityDownloadRequest - { - BucketName = bucketName, - Key = fileName, - FilePath = $"{temporalPath}{fileName}", - }; - - using var transferUtility = new TransferUtility(_s3Client); - await transferUtility.DownloadAsync(downloadRequest); - } + var copy = await _s3Client.CopyObjectAsync(sourceBucket, sourceKey, targetBucket, targetKey); + return targetKey; + } /// /// Delete file from s3 /// /// bucket name - /// file name /key + /// s3 key /key /// DeleteFileResponse - public async Task DeleteFile(string bucketName, string fileName) + public async Task DeleteFile(string bucketName, string key) { - var multiObjectDeleteRequest = new DeleteObjectsRequest - { - BucketName = bucketName - }; + var multiObjectDeleteRequest = new DeleteObjectsRequest + { + BucketName = bucketName + }; - multiObjectDeleteRequest.AddKey(fileName); + multiObjectDeleteRequest.AddKey(key); + + var response = await _s3Client.DeleteObjectsAsync(multiObjectDeleteRequest); + + return new DeleteFileResponse + { + NumberOfDeletedObjects = response.DeletedObjects.Count + }; + } - var response = await _s3Client.DeleteObjectsAsync(multiObjectDeleteRequest); + /// + /// Download file from s3 to local temp folder + /// + /// bucket name + /// key = path + file name + /// temporal path to download + file name + /// task just to download file + public async Task DownloadFile(string bucketName, string key, string temporalPath) + { + GetObjectRequest request = new GetObjectRequest + { + BucketName = bucketName, + Key = key + }; - return new DeleteFileResponse + var downloadRequest = new TransferUtilityDownloadRequest { - NumberOfDeletedObjects = response.DeletedObjects.Count + BucketName = bucketName, + Key = key, + FilePath = temporalPath, }; + + using var transferUtility = new TransferUtility(_s3Client); + await transferUtility.DownloadAsync(downloadRequest); } /// - /// Add json object + /// Get file from s3 bucket /// /// bucket name - /// object request - public async Task AddJsonObject(string bucketName, AddJsonObjectRequest request) + /// s3 key - path + file name + /// Memory file object + public async Task DownloadMemoryStreamAsync(string bucketName, string key) { - var createdOnUtc = DateTime.UtcNow; + var memory = new MemoryStream(); + try + { + var streamRequest = new TransferUtilityOpenStreamRequest + { + BucketName = bucketName, + Key = key + }; - var s3Key = $"{createdOnUtc:yyyy}/{createdOnUtc:MM}/{createdOnUtc:dd}/{request.Id}"; + var request = new GetObjectRequest() + { + BucketName = bucketName, + Key = key + }; - var putObjectRequest = new PutObjectRequest + using (var transferUtility = new TransferUtility(_s3Client)) + { + var objectResponse = await transferUtility.S3Client.GetObjectAsync(request); + + var stream = objectResponse.ResponseStream; + + await stream.CopyToAsync(memory); + } + + memory.Position = 0; + + return memory; + } + catch (AmazonS3Exception e) { - BucketName = bucketName, - Key = s3Key, - ContentBody = JsonConvert.SerializeObject(request) - }; + Console.Write("Error encountered on server. Message:'{0}' when writing an object", e.Message); + } + catch (Exception e) + { + Console.Write("Download fail", e.Message); + } - await _s3Client.PutObjectAsync(putObjectRequest); + return memory; } /// /// Get json object /// /// bucket name - /// file name + /// path + file name /// GetJsonObjectResponse - public async Task GetJsonObject(string bucketName, string fileName) + public async Task GetJsonObject(string bucketName, string key) { var request = new GetObjectRequest { BucketName = bucketName, - Key = fileName + Key = key }; var response = await _s3Client.GetObjectAsync(request); - using (var reader = new StreamReader(response.ResponseStream)) - { - var contents = reader.ReadToEnd(); - return JsonConvert.DeserializeObject(contents); - } + using var reader = new StreamReader(response.ResponseStream); + var contents = await reader.ReadToEndAsync(); + return JsonConvert.DeserializeObject(contents); + } + + /// + /// Generate presigned url + /// + /// bucket name + /// file key + /// time to expire url + /// + public AddSingleFileResponse GetPreSignedUrl(string bucketName, string key, DateTime expirationTime) + { + var expiryUrlRequest = new GetPreSignedUrlRequest + { + BucketName = bucketName, + Key = key, + Expires = expirationTime + }; + + var url = _s3Client.GetPreSignedURL(expiryUrlRequest); + + var response = new AddSingleFileResponse() + { + PreSignedUrl = url, // temporal download url + Key = key // Internal reference s3 key + }; + + return response; } } } \ No newline at end of file diff --git a/FacwareBase.API/appsettings.Development.json b/FacwareBase.API/appsettings.Development.json index 5d37d63..1610a2a 100644 --- a/FacwareBase.API/appsettings.Development.json +++ b/FacwareBase.API/appsettings.Development.json @@ -76,8 +76,8 @@ "ExpirationInHours": 4 }, "AWS": { - "Profile": "local-test-profile", - "Region": "ap-southeast-1" + "Profile": "my-profile", + "Region": "us-east-1" }, "SessionAwsCredentialsOptions": { "AwsAccessKeyId": "", diff --git a/FacwareBase.API/appsettings.Local.json b/FacwareBase.API/appsettings.Local.json index 219363c..5261d84 100644 --- a/FacwareBase.API/appsettings.Local.json +++ b/FacwareBase.API/appsettings.Local.json @@ -74,12 +74,16 @@ "ExpirationInDays": 30, "ExpirationInHours": 4 }, + "AWS": { + "Profile": "my-profile", + "Region": "us-east-1" + }, "SessionAwsCredentialsOptions": { - "AwsAccessKeyId": "", - "AwsSecretAccessKey": "", - "Token": "" + "AwsAccessKeyId": "", + "AwsSecretAccessKey": "", + "Token": "" }, "FileStorageOptions": { - "TemporalStorage": "/tmp/" + "TemporalStorage": "/tmp/" } } diff --git a/FacwareBase.API/appsettings.Staging.json b/FacwareBase.API/appsettings.Staging.json index c1f1644..32d3755 100644 --- a/FacwareBase.API/appsettings.Staging.json +++ b/FacwareBase.API/appsettings.Staging.json @@ -76,12 +76,16 @@ "ExpirationInDays": 30, "ExpirationInHours": 4 }, + "AWS": { + "Profile": "my-profile", + "Region": "us-east-1" + }, "SessionAwsCredentialsOptions": { - "AwsAccessKeyId": "", - "AwsSecretAccessKey": "", - "Token": "" + "AwsAccessKeyId": "", + "AwsSecretAccessKey": "", + "Token": "" }, "FileStorageOptions": { - "TemporalStorage": "/tmp/" + "TemporalStorage": "/tmp/" } } diff --git a/Readme.md b/Readme.md index 5c405b7..a230888 100644 --- a/Readme.md +++ b/Readme.md @@ -43,7 +43,7 @@ Use this attribute `EnableQueryFromODataToAWS` instead `EnableQuery` ### AWS lambda Create API build -`dotnet publish -c Release -o AWSLambda` +`demo` Handler `FacwareBase.API::FacwareBase.API.LambdaEntryPoint::FunctionHandlerAsync`