From d779259290a04bcb69240ec8f53fdcb461dca506 Mon Sep 17 00:00:00 2001 From: Stephane Royer Date: Thu, 16 Jan 2025 18:04:03 +0100 Subject: [PATCH] feat: implement zip and unzip functionality with new classes and parameters --- .../ZipFileProcessorValuesProvider.cs | 106 +++++++++--------- .../ZipProviderProcessorAdapter.cs | 80 +++++++------ src/Paillave.Etl.Zip/ZippedFileValue.cs | 34 ++++++ src/SharedSettings.props | 2 +- 4 files changed, 139 insertions(+), 83 deletions(-) create mode 100644 src/Paillave.Etl.Zip/ZippedFileValue.cs diff --git a/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs b/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs index a061e7d0..f00f6410 100644 --- a/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs +++ b/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs @@ -1,52 +1,58 @@ -// using System; -// using System.Linq; -// using System.IO; -// using ICSharpCode.SharpZipLib.Zip; -// using Paillave.Etl.Core; -// using System.Threading; -// using Microsoft.Extensions.FileSystemGlobbing; +using System; +using System.Linq; +using System.IO; +using ICSharpCode.SharpZipLib.Zip; +using Paillave.Etl.Core; +using System.Threading; +using Microsoft.Extensions.FileSystemGlobbing; +using System.Collections.Generic; -// namespace Paillave.Etl.Zip -// { -// public class ZipFileProcessorValuesProvider : ValuesProviderBase -// { -// private ZipFileProcessorParams _args; -// public ZipFileProcessorValuesProvider(UnzipFileProcessorParams args) -// => _args = args; -// public override ProcessImpact PerformanceImpact => ProcessImpact.Average; -// public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; -// public override void PushValues(IFileValue input, Action push, CancellationToken cancellationToken, IExecutionContext context) -// { -// var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations; -// if (cancellationToken.IsCancellationRequested) return; -// using var stream = input.Get(_args.UseStreamCopy); -// using var zf = new ZipFile(stream); -// var searchPattern = string.IsNullOrEmpty(_args.FileNamePattern) ? "*" : _args.FileNamePattern; -// var matcher = new Matcher().AddInclude(searchPattern); +namespace Paillave.Etl.Zip; -// if (!String.IsNullOrEmpty(_args.Password)) -// zf.Password = _args.Password; -// var fileNames = zf.OfType().Where(i => i.IsFile && matcher.Match(Path.GetFileName(i.Name)).HasMatches).Select(i => i.Name).ToHashSet(); -// foreach (ZipEntry zipEntry in zf) -// { -// if (cancellationToken.IsCancellationRequested) break; -// if (zipEntry.IsFile && matcher.Match(Path.GetFileName(zipEntry.Name)).HasMatches) -// { -// MemoryStream outputStream = new MemoryStream(); -// using (var zipStream = zf.GetInputStream(zipEntry)) -// zipStream.CopyTo(outputStream, 4096); -// outputStream.Seek(0, SeekOrigin.Begin); -// push(new UnzippedFileValue(outputStream, zipEntry.Name, new UnzippedFileValueMetadata -// { -// ParentFileName = input.Name, -// ParentFileMetadata = input.Metadata, -// Destinations = destinations, -// ConnectorCode = input.Metadata.ConnectorCode, -// ConnectionName = input.Metadata.ConnectionName, -// ConnectorName = input.Metadata.ConnectorName -// }, input, fileNames, zipEntry.Name)); -// } -// } -// } -// } -// } +public class ZipFileProcessorParams +{ + public string Password { get; set; } + public bool UseStreamCopy { get; set; } = true; +} +public class ZippedFileValueMetadata : FileValueMetadataBase, IFileValueWithDestinationMetadata +{ + public IFileValueMetadata ParentFileMetadata { get; set; } + public Dictionary> Destinations { get; set; } +} +public class ZipFileProcessorValuesProvider : ValuesProviderBase +{ + private ZipFileProcessorParams _args; + public ZipFileProcessorValuesProvider(ZipFileProcessorParams args) + => _args = args; + public override ProcessImpact PerformanceImpact => ProcessImpact.Average; + public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; + public override void PushValues(IFileValue input, Action push, CancellationToken cancellationToken, IExecutionContext context) + { + var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations; + if (cancellationToken.IsCancellationRequested) return; + using var stream = input.Get(_args.UseStreamCopy); + var ms = new MemoryStream(); + var fileName = $"{input.Name}.zip"; + using (ZipOutputStream zipStream = new ZipOutputStream(ms)) + { + if (!String.IsNullOrEmpty(_args.Password)) + zipStream.Password = _args.Password; + + var zipEntry = new ZipEntry(fileName) + { + DateTime = DateTime.Now, + IsUnicodeText = true + }; + + zipStream.PutNextEntry(zipEntry); + stream.CopyTo(zipStream); + zipStream.CloseEntry(); + } + ms.Seek(0, SeekOrigin.Begin); + push(new ZippedFileValue(ms, input.Name, new ZippedFileValueMetadata + { + ParentFileMetadata = input.Metadata, + Destinations = destinations + }, input)); + } +} diff --git a/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs b/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs index a023028c..0b6067d1 100644 --- a/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs +++ b/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs @@ -2,40 +2,56 @@ using System.Threading; using Paillave.Etl.Core; -namespace Paillave.Etl.Zip +namespace Paillave.Etl.Zip; + +public class ZipAdapterConnectionParameters { - public class ZipAdapterConnectionParameters - { - public string Password { get; set; } - } - public class ZipAdapterProcessorParameters - { - public string FileNamePattern { get; set; } - } - public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase - { - public override string Description => "Handle zip files"; - public override string Name => "Zip"; - protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters) - => null; - protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters) - => new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters); - } - public class ZipFileValueProcessor : FileValueProcessorBase + public string Password { get; set; } +} +public enum ZipDirection +{ + Unzip, + Zip +} +public class ZipAdapterProcessorParameters +{ + public ZipDirection Direction { get; set; } = ZipDirection.Unzip; + public string FileNamePattern { get; set; } +} +public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase +{ + public override string Description => "Handle zip files"; + public override string Name => "Zip"; + protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters) + => null; + protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters) + => new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters); +} +public class ZipFileValueProcessor : FileValueProcessorBase +{ + public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) + : base(code, name, connectionName, connectionParameters, processorParameters) { } + public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy; + public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; + protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action push, CancellationToken cancellationToken, IExecutionContext context) { - public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) - : base(code, name, connectionName, connectionParameters, processorParameters) { } - public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy; - public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; - protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action push, CancellationToken cancellationToken, IExecutionContext context) + switch (processorParameters.Direction) { - new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams - { - FileNamePattern = processorParameters.FileNamePattern, - Password = connectionParameters.Password - }).PushValues(fileValue, push, cancellationToken, context); + case ZipDirection.Unzip: + new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams + { + FileNamePattern = processorParameters.FileNamePattern, + Password = connectionParameters.Password + }).PushValues(fileValue, push, cancellationToken, context); + break; + case ZipDirection.Zip: + new ZipFileProcessorValuesProvider(new ZipFileProcessorParams + { + Password = connectionParameters.Password + }).PushValues(fileValue, push, cancellationToken, context); + break; } - - protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { } } -} \ No newline at end of file + + protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { } +} diff --git a/src/Paillave.Etl.Zip/ZippedFileValue.cs b/src/Paillave.Etl.Zip/ZippedFileValue.cs new file mode 100644 index 00000000..cc080a16 --- /dev/null +++ b/src/Paillave.Etl.Zip/ZippedFileValue.cs @@ -0,0 +1,34 @@ +using System.IO; +using Paillave.Etl.Core; + +namespace Paillave.Etl.Zip +{ + public class ZippedFileValue : FileValueBase where TMetadata : IFileValueMetadata + { + private readonly Stream _stream; + private readonly IFileValue _underlyingFileValue; + public override string Name { get; } + public ZippedFileValue(Stream stream, string name, TMetadata metadata, IFileValue underlyingFileValue) + : base(metadata) + => (_stream, Name, _underlyingFileValue) + = (stream, name, underlyingFileValue); + public override Stream GetContent() + { + var ms = new MemoryStream(); + _stream.Seek(0, SeekOrigin.Begin); + _stream.CopyTo(ms); + ms.Seek(0, SeekOrigin.Begin); + return ms; + } + protected override void DeleteFile() + { + _underlyingFileValue.Delete(); + } + + public override StreamWithResource OpenContent() + { + _stream.Seek(0, SeekOrigin.Begin); + return new StreamWithResource(_stream); + } + } +} \ No newline at end of file diff --git a/src/SharedSettings.props b/src/SharedSettings.props index 1f669070..f58a425b 100644 --- a/src/SharedSettings.props +++ b/src/SharedSettings.props @@ -2,7 +2,7 @@ latest enable - 2.2.5-beta + 2.2.6-beta NugetIcon.png README.md Stéphane Royer