feat: implement zip and unzip functionality with new classes and para…

…meters
paillave · Jan 16, 2025 · d779259 · d779259
1 parent 6a3bd86
commit d779259
Show file tree

Hide file tree

Showing 4 changed files with 139 additions and 83 deletions.
diff --git a/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs b/src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs
@@ -1,52 +1,58 @@
-// using System;
-// using System.Linq;
-// using System.IO;
-// using ICSharpCode.SharpZipLib.Zip;
-// using Paillave.Etl.Core;
-// using System.Threading;
-// using Microsoft.Extensions.FileSystemGlobbing;
+using System;
+using System.Linq;
+using System.IO;
+using ICSharpCode.SharpZipLib.Zip;
+using Paillave.Etl.Core;
+using System.Threading;
+using Microsoft.Extensions.FileSystemGlobbing;
+using System.Collections.Generic;
 
-// namespace Paillave.Etl.Zip
-// {
-//     public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
-//     {
-//         private ZipFileProcessorParams _args;
-//         public ZipFileProcessorValuesProvider(UnzipFileProcessorParams args)
-//             => _args = args;
-//         public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
-//         public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
-//         public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
-//         {
-//             var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
-//             if (cancellationToken.IsCancellationRequested) return;
-//             using var stream = input.Get(_args.UseStreamCopy);
-//             using var zf = new ZipFile(stream);
-//             var searchPattern = string.IsNullOrEmpty(_args.FileNamePattern) ? "*" : _args.FileNamePattern;
-//             var matcher = new Matcher().AddInclude(searchPattern);
+namespace Paillave.Etl.Zip;
 
-//             if (!String.IsNullOrEmpty(_args.Password))
-//                 zf.Password = _args.Password;
-//             var fileNames = zf.OfType<ZipEntry>().Where(i => i.IsFile && matcher.Match(Path.GetFileName(i.Name)).HasMatches).Select(i => i.Name).ToHashSet();
-//             foreach (ZipEntry zipEntry in zf)
-//             {
-//                 if (cancellationToken.IsCancellationRequested) break;
-//                 if (zipEntry.IsFile && matcher.Match(Path.GetFileName(zipEntry.Name)).HasMatches)
-//                 {
-//                     MemoryStream outputStream = new MemoryStream();
-//                     using (var zipStream = zf.GetInputStream(zipEntry))
-//                         zipStream.CopyTo(outputStream, 4096);
-//                     outputStream.Seek(0, SeekOrigin.Begin);
-//                     push(new UnzippedFileValue<UnzippedFileValueMetadata>(outputStream, zipEntry.Name, new UnzippedFileValueMetadata
-//                     {
-//                         ParentFileName = input.Name,
-//                         ParentFileMetadata = input.Metadata,
-//                         Destinations = destinations,
-//                         ConnectorCode = input.Metadata.ConnectorCode,
-//                         ConnectionName = input.Metadata.ConnectionName,
-//                         ConnectorName = input.Metadata.ConnectorName
-//                     }, input, fileNames, zipEntry.Name));
-//                 }
-//             }
-//         }
-//     }
-// }
+public class ZipFileProcessorParams
+{
+    public string Password { get; set; }
+    public bool UseStreamCopy { get; set; } = true;
+}
+public class ZippedFileValueMetadata : FileValueMetadataBase, IFileValueWithDestinationMetadata
+{
+    public IFileValueMetadata ParentFileMetadata { get; set; }
+    public Dictionary<string, IEnumerable<Destination>> Destinations { get; set; }
+}
+public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
+{
+    private ZipFileProcessorParams _args;
+    public ZipFileProcessorValuesProvider(ZipFileProcessorParams args)
+        => _args = args;
+    public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
+    public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
+    public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
+    {
+        var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
+        if (cancellationToken.IsCancellationRequested) return;
+        using var stream = input.Get(_args.UseStreamCopy);
+        var ms = new MemoryStream();
+        var fileName = $"{input.Name}.zip";
+        using (ZipOutputStream zipStream = new ZipOutputStream(ms))
+        {
+            if (!String.IsNullOrEmpty(_args.Password))
+                zipStream.Password = _args.Password;
+
+            var zipEntry = new ZipEntry(fileName)
+            {
+                DateTime = DateTime.Now,
+                IsUnicodeText = true
+            };
+
+            zipStream.PutNextEntry(zipEntry);
+            stream.CopyTo(zipStream);
+            zipStream.CloseEntry();
+        }
+        ms.Seek(0, SeekOrigin.Begin);
+        push(new ZippedFileValue<ZippedFileValueMetadata>(ms, input.Name, new ZippedFileValueMetadata
+        {
+            ParentFileMetadata = input.Metadata,
+            Destinations = destinations
+        }, input));
+    }
+}
diff --git a/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs b/src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs
@@ -2,40 +2,56 @@
 using System.Threading;
 using Paillave.Etl.Core;
 
-namespace Paillave.Etl.Zip
+namespace Paillave.Etl.Zip;
+
+public class ZipAdapterConnectionParameters
 {
-    public class ZipAdapterConnectionParameters
-    {
-        public string Password { get; set; }
-    }
-    public class ZipAdapterProcessorParameters
-    {
-        public string FileNamePattern { get; set; }
-    }
-    public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
-    {
-        public override string Description => "Handle zip files";
-        public override string Name => "Zip";
-        protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
-            => null;
-        protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
-            => new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
-    }
-    public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
+    public string Password { get; set; }
+}
+public enum ZipDirection
+{
+    Unzip,
+    Zip
+}
+public class ZipAdapterProcessorParameters
+{
+    public ZipDirection Direction { get; set; } = ZipDirection.Unzip;
+    public string FileNamePattern { get; set; }
+}
+public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
+{
+    public override string Description => "Handle zip files";
+    public override string Name => "Zip";
+    protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
+        => null;
+    protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
+        => new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
+}
+public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
+{
+    public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
+        : base(code, name, connectionName, connectionParameters, processorParameters) { }
+    public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
+    public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
+    protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
     {
-        public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
-            : base(code, name, connectionName, connectionParameters, processorParameters) { }
-        public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
-        public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
-        protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
+        switch (processorParameters.Direction)
         {
-            new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
-            {
-                FileNamePattern = processorParameters.FileNamePattern,
-                Password = connectionParameters.Password
-            }).PushValues(fileValue, push, cancellationToken, context);
+            case ZipDirection.Unzip:
+                new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
+                {
+                    FileNamePattern = processorParameters.FileNamePattern,
+                    Password = connectionParameters.Password
+                }).PushValues(fileValue, push, cancellationToken, context);
+                break;
+            case ZipDirection.Zip:
+                new ZipFileProcessorValuesProvider(new ZipFileProcessorParams
+                {
+                    Password = connectionParameters.Password
+                }).PushValues(fileValue, push, cancellationToken, context);
+                break;
         }
-
-        protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
     }
-}
+
+    protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
+}
diff --git a/src/Paillave.Etl.Zip/ZippedFileValue.cs b/src/Paillave.Etl.Zip/ZippedFileValue.cs
@@ -0,0 +1,34 @@
+using System.IO;
+using Paillave.Etl.Core;
+
+namespace Paillave.Etl.Zip
+{
+    public class ZippedFileValue<TMetadata> : FileValueBase<TMetadata> where TMetadata : IFileValueMetadata
+    {
+        private readonly Stream _stream;
+        private readonly IFileValue _underlyingFileValue;
+        public override string Name { get; }
+        public ZippedFileValue(Stream stream, string name, TMetadata metadata, IFileValue underlyingFileValue)
+            : base(metadata)
+            => (_stream, Name, _underlyingFileValue)
+            = (stream, name, underlyingFileValue);
+        public override Stream GetContent()
+        {
+            var ms = new MemoryStream();
+            _stream.Seek(0, SeekOrigin.Begin);
+            _stream.CopyTo(ms);
+            ms.Seek(0, SeekOrigin.Begin);
+            return ms;
+        }
+        protected override void DeleteFile()
+        {
+            _underlyingFileValue.Delete();
+        }
+
+        public override StreamWithResource OpenContent()
+        {
+            _stream.Seek(0, SeekOrigin.Begin);
+            return new StreamWithResource(_stream);
+        }
+    }
+}
diff --git a/src/SharedSettings.props b/src/SharedSettings.props
@@ -2,7 +2,7 @@
   <PropertyGroup>
     <LangVersion>latest</LangVersion>
     <Nullable>enable</Nullable>
-    <Version>2.2.5-beta</Version>
+    <Version>2.2.6-beta</Version>
     <PackageIcon>NugetIcon.png</PackageIcon>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <Authors>Stéphane Royer</Authors>