-
-
Notifications
You must be signed in to change notification settings - Fork 96
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #450 from paillave/Scheduler
GraphApi
- Loading branch information
Showing
71 changed files
with
2,868 additions
and
302 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,14 @@ | |
| [](https://paillave.github.io/Etl.Net/) | ||
| [Go to full documentation](https://paillave.github.io/Etl.Net/) | ||
|
|
||
| ## :rotating_light: I need some help!!! :ambulance: | ||
|
|
||
| I need some serious help for the following subject: | ||
| - :books: Documentation. The [website of ETL.NET](https://paillave.github.io/Etl.Net/) needs to be completed. It contains the essential to start and get in touch, but a dramatic amount of serious features is not documented in there. | ||
| - :heavy_check_mark: Unit tests. A proper test code coverage is the only real way to be taken seriously when it is about selecting the right open source library. There was a time when some units tests existed only for the reactive engine of the core (Paillave.EtlNet.Core). But it happened I had to decommission it in the even of the V2 release. | ||
|
|
||
| :envelope_with_arrow: Anybody who is keen to participate to the project in these regards is very welcome to contact me at [email protected]. | ||
|
|
||
| ## Presentation | ||
|
|
||
| Implementation of a mass processing engine to use in a similar way than Linq with every SSIS features and much more. | ||
|
|
@@ -24,6 +32,7 @@ Extend it takes 5mn... literally. | |
| | Paillave.EtlNet.FromConfigurationConnectors | [](https://www.nuget.org/packages/Paillave.EtlNet.FromConfigurationConnectors) | [](https://www.nuget.org/packages/Paillave.EtlNet.FromConfigurationConnectors) | | ||
| | Paillave.EtlNet.Ftp | [](https://www.nuget.org/packages/Paillave.EtlNet.Ftp) | [](https://www.nuget.org/packages/Paillave.EtlNet.Ftp) | | ||
| | Paillave.EtlNet.Mail | [](https://www.nuget.org/packages/Paillave.EtlNet.Mail) | [](https://www.nuget.org/packages/Paillave.EtlNet.Mail) | | ||
| | Paillave.EtlNet.GraphApi | [](https://www.nuget.org/packages/Paillave.EtlNet.GraphApi) | [](https://www.nuget.org/packages/Paillave.EtlNet.GraphApi) | | ||
| | Paillave.EtlNet.Sftp | [](https://www.nuget.org/packages/Paillave.EtlNet.Sftp) | [](https://www.nuget.org/packages/Paillave.EtlNet.Sftp) | | ||
| | Paillave.EtlNet.SqlServer | [](https://www.nuget.org/packages/Paillave.EtlNet.SqlServer) | [](https://www.nuget.org/packages/Paillave.EtlNet.SqlServer) | | ||
| | Paillave.EtlNet.TextFile | [](https://www.nuget.org/packages/Paillave.EtlNet.TextFile) | [](https://www.nuget.org/packages/Paillave.EtlNet.TextFile) | | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Graph; | ||
| using Microsoft.Graph.Models; | ||
| using Microsoft.Graph.Users.Item.MailFolders; | ||
| using Paillave.Etl.GraphApi.Provider; | ||
|
|
||
| namespace Paillave.Etl.GraphApi; | ||
|
|
||
| public static class BaseGraphServiceClientEx | ||
| { | ||
| public static async Task<MailFolder> GetFolderAsync(this BaseGraphServiceClient graphClient, string userId, string path, CancellationToken cancellationToken) | ||
| { | ||
| var segments = path.Split(new char[] { '/', '\\' }); | ||
| var folder = await GetRootFolderAsync(graphClient, userId, segments[0], cancellationToken); | ||
| if (folder == null) | ||
| throw new Exception("folder does not exist"); | ||
| foreach (var segment in segments.Skip(1)) | ||
| { | ||
| var tmp = await graphClient.Users[userId].MailFolders[folder.Id] | ||
| .GetAsync(i => i.QueryParameters.Expand = ProjectionProcessor<MailFolder>.ToString(j => j.ChildFolders)); | ||
| if (tmp == null) | ||
| throw new Exception("this is quiet unexpected"); | ||
| folder = tmp.ChildFolders?.FirstOrDefault(i => i.DisplayName == segment); | ||
| if (folder == null) | ||
| throw new Exception("folder does not exist"); | ||
| } | ||
| return folder; | ||
| } | ||
|
|
||
| public static async Task<MailFolder?> GetRootFolderAsync(this BaseGraphServiceClient graphClient, string userId, string rootFolderName, CancellationToken cancellationToken) | ||
| { | ||
| MailFoldersRequestBuilder mailFoldersRequestBuilder = graphClient.Users[userId].MailFolders; | ||
| var response = await mailFoldersRequestBuilder.GetAsync(i => | ||
| { | ||
| i.QueryParameters.Filter = ODataExpression<MailFolder>.ToString(j => j.DisplayName == rootFolderName); | ||
| i.QueryParameters.Expand = ProjectionProcessor<MailFolder>.ToString(j => j.ChildFolders); | ||
| i.QueryParameters.Top = 1; | ||
| }, cancellationToken); | ||
| return response.Value.FirstOrDefault(); | ||
| } | ||
| } |
36 changes: 36 additions & 0 deletions
36
src/Paillave.Etl.GraphApi/GraphApiAdapterConnectionParametersEx.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| using System.Collections.Generic; | ||
| using System.Net.Http.Headers; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Graph; | ||
| using Microsoft.Identity.Client; | ||
| using Microsoft.Kiota.Abstractions; | ||
| using Microsoft.Kiota.Abstractions.Authentication; | ||
|
|
||
| namespace Paillave.Etl.GraphApi; | ||
|
|
||
| public static class GraphApiAdapterConnectionParametersEx | ||
| { | ||
| public static GraphServiceClient CreateGraphApiClient(this IGraphApiConnectionInfo connectionInfo) | ||
| { | ||
| var confidentialClientApplication = ConfidentialClientApplicationBuilder | ||
| .Create(connectionInfo.ClientId) | ||
| .WithTenantId(connectionInfo.TenantId) | ||
| .WithClientSecret(connectionInfo.ClientSecret) | ||
| .Build(); | ||
|
|
||
| var graphClient = new GraphServiceClient(new CustomAuthenticationProvider(confidentialClientApplication)); | ||
|
|
||
| return graphClient; | ||
| } | ||
| } | ||
| internal class CustomAuthenticationProvider : IAuthenticationProvider | ||
| { | ||
| private readonly IConfidentialClientApplication _app; | ||
| public CustomAuthenticationProvider(IConfidentialClientApplication app) => _app = app; | ||
| public async Task AuthenticateRequestAsync(RequestInformation request, Dictionary<string, object>? additionalAuthenticationContext = null, CancellationToken cancellationToken = default) | ||
| { | ||
| var result = await _app.AcquireTokenForClient(new string[] { "https://graph.microsoft.com/.default" }).ExecuteAsync(); | ||
| request.Headers.Add("Authorization", new AuthenticationHeaderValue("Bearer", result.AccessToken).ToString()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| using Microsoft.Graph.Models; | ||
| using Paillave.Etl.Core; | ||
| using Paillave.Etl.GraphApi.Provider; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
|
|
||
| namespace Paillave.Etl.GraphApi; | ||
|
|
||
| public class GraphApiMailFileValue : FileValueBase<GraphApiMailFileValueMetadata> | ||
| { | ||
| public override string Name { get; } | ||
| private readonly string _userId; | ||
| private readonly string _messageId; | ||
| private readonly string _attachmentId; | ||
| private bool _setToReadIfBatchDeletion; | ||
| private readonly IGraphApiConnectionInfo _connectionInfo; | ||
| private readonly Dictionary<string, bool> _deleted; | ||
| internal GraphApiMailFileValue( | ||
| IGraphApiConnectionInfo connectionInfo, | ||
| string messageId, string attachmentId, | ||
| string folder, string mailSubject, string attachmentName, | ||
| string sender, DateTime receivedDate, | ||
| string connectorCode, string connectorName, string connectionName, | ||
| bool setToReadIfBatchDeletion, Dictionary<string, bool> deleted) | ||
| : base(new GraphApiMailFileValueMetadata | ||
| { | ||
| ClientId = connectionInfo.ClientId, | ||
| TenantId = connectionInfo.TenantId, | ||
|
|
||
| UserId = connectionInfo.UserId, | ||
| MessageId = messageId, | ||
| AttachmentId = attachmentId, | ||
|
|
||
| Folder = folder, | ||
| MailSubject = mailSubject, | ||
| Name = attachmentName, | ||
|
|
||
| Sender = sender, | ||
| ReceivedDate = receivedDate, | ||
|
|
||
| ConnectorCode = connectorCode, | ||
| ConnectionName = connectionName, | ||
| ConnectorName = connectorName, | ||
| }) => (Name, _userId, _messageId, _attachmentId, _connectionInfo, _deleted, _setToReadIfBatchDeletion) | ||
| = (attachmentName, connectionInfo.UserId, messageId, attachmentId, connectionInfo, deleted, setToReadIfBatchDeletion); | ||
| protected override void DeleteFile() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, DeleteFileSingleTime); | ||
| protected void DeleteFileSingleTime() | ||
| { | ||
| _deleted[this._attachmentId] = true; | ||
| if (_deleted.Values.All(v => v)) | ||
| { | ||
| using (var client = _connectionInfo.CreateGraphApiClient()) | ||
| { | ||
| var messageRequestBuilder = client.Users[_userId].Messages[_messageId]; | ||
| if (_setToReadIfBatchDeletion) | ||
| { | ||
| var messageToDelete = messageRequestBuilder.GetAsync(i => | ||
| { | ||
| i.QueryParameters.Select = ProjectionProcessor<Message>.ToString(m => new { m.IsRead, m.Id }); | ||
| }).Result; | ||
| messageToDelete.IsRead = true; | ||
| messageRequestBuilder.PatchAsync(messageToDelete); | ||
| } | ||
| else | ||
| { | ||
| messageRequestBuilder.DeleteAsync().Wait(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| public override Stream GetContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, GetContentSingleTime); | ||
| private Stream GetContentSingleTime() | ||
| { | ||
| using (var graphClient = _connectionInfo.CreateGraphApiClient()) | ||
| { | ||
| var fullMessageRequestBuilder = graphClient.Users[_userId].Messages[_messageId]; | ||
|
|
||
| var attachment = (FileAttachment)fullMessageRequestBuilder.Attachments[_attachmentId].GetAsync().Result; | ||
| var ms = new MemoryStream(); | ||
| ms.Write(attachment?.ContentBytes ?? new byte[] { }); | ||
| ms.Seek(0, SeekOrigin.Begin); | ||
| return ms; | ||
| } | ||
| } | ||
|
|
||
| public override StreamWithResource OpenContent() => new StreamWithResource(GetContent()); | ||
| } | ||
| public class GraphApiMailFileValueMetadata : FileValueMetadataBase | ||
| { | ||
| public string TenantId { get; set; } | ||
| public string ClientId { get; set; } | ||
| public string UserId { get; set; } | ||
| public string MessageId { get; set; } | ||
| public string AttachmentId { get; set; } | ||
| public string Folder { get; set; } | ||
| public string Name { get; set; } | ||
| public string MailSubject { get; set; } | ||
| public DateTime ReceivedDate { get; set; } | ||
| public string Sender { get; set; } | ||
| } |
Oops, something went wrong.