Skip to content

Commit

Permalink
Merge pull request #1677 from qdraw/feature/202408_exiftool_dl_fix
Browse files Browse the repository at this point in the history 
fix for exiftool download
  • Loading branch information
@qdraw
qdraw authored Aug 30, 2024
2 parents 6f8d47c + 1cfb463 commit df12b47
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 135 deletions.
16 changes: 8 additions & 8 deletions starsky/starsky.foundation.http/Services/HttpClientHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,22 +49,22 @@ public HttpClientHelper(IHttpProvider httpProvider,
private readonly IWebLogger _logger;

/// <summary>
/// This domains are only allowed domains to download from (and https only)
/// These domains are only allowed domains to download from (and https only)
/// </summary>
private readonly List<string> _allowedDomains = new List<string>
{
private readonly List<string> _allowedDomains =
[
"dl.dropboxusercontent.com",
"qdraw.nl", // < used by test
"media.qdraw.nl", // < used by demo
"locker.ifttt.com",
"media.qdraw.nl", // < used by demo
"locker.ifttt.com",
"download.geonames.org",
"exiftool.org",
"api.github.com"
};
];

public async Task<KeyValuePair<bool, string>> ReadString(string sourceHttpUrl)
{
Uri sourceUri = new Uri(sourceHttpUrl);
var sourceUri = new Uri(sourceHttpUrl);

_logger.LogInformation("[ReadString] HttpClientHelper > "
+ sourceUri.Host + " ~ " + sourceHttpUrl);
Expand Down Expand Up @@ -92,7 +92,7 @@ public async Task<KeyValuePair<bool, string>> ReadString(string sourceHttpUrl)
public async Task<KeyValuePair<bool, string>> PostString(string sourceHttpUrl,
HttpContent? httpContent, bool verbose = true)
{
Uri sourceUri = new Uri(sourceHttpUrl);
var sourceUri = new Uri(sourceHttpUrl);

if ( verbose ) _logger.LogInformation("[PostString] HttpClientHelper > "
+ sourceUri.Host + " ~ " + sourceHttpUrl);
Expand Down
145 changes: 97 additions & 48 deletions starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
Expand All @@ -24,7 +23,6 @@ namespace starsky.foundation.writemeta.Services
{
[Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)]
[SuppressMessage("Usage", "S1075:Refactor your code not to use hardcoded absolute paths or URIs", Justification = "Source of files")]
[SuppressMessage("Usage", "S4790:Make sure this weak hash algorithm is not used in a sensitive context here.", Justification = "Safe")]
public sealed class ExifToolDownload : IExifToolDownload
{
private readonly IHttpClientHelper _httpClientHelper;
Expand Down Expand Up @@ -56,13 +54,12 @@ internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSet
/// <summary>
/// Auto Download Exiftool
/// </summary>
/// <param name="isWindows">download windows version if true</param>
/// <param name="isWindows">download Windows version if true</param>
/// <param name="minimumSize">check for min file size in bytes (Default = 30 bytes)</param>
/// <returns></returns>
public async Task<bool> DownloadExifTool(bool isWindows, int minimumSize = 30)
{
if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || (
_appSettings.AddSwaggerExport == true && _appSettings.AddSwaggerExportExitAfter == true ) )
if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } )
{
var name = _appSettings.ExiftoolSkipDownloadOnStartup == true
? "ExiftoolSkipDownloadOnStartup"
Expand Down Expand Up @@ -102,42 +99,53 @@ public async Task<bool> DownloadExifTool(bool isWindows, int minimumSize = 30)

private void CreateDirectoryDependenciesFolderIfNotExists()
{
if ( _hostFileSystemStorage.ExistFolder(_appSettings
.DependenciesFolder) ) return;
if ( _hostFileSystemStorage.ExistFolder(
_appSettings.DependenciesFolder) )
{
return;
}
_logger.LogInformation("[DownloadExifTool] Create Directory: " + _appSettings.DependenciesFolder);
_hostFileSystemStorage.CreateDirectory(_appSettings.DependenciesFolder);
}

internal async Task<KeyValuePair<bool, string>?> DownloadCheckSums()
{
var checksums = await _httpClientHelper.ReadString(CheckSumLocation);
var baseLocationResult = await DownloadCheckSums(CheckSumLocation);
if ( baseLocationResult == null )
{
return await DownloadCheckSums(CheckSumLocationMirror);
}
return baseLocationResult;
}

internal async Task<KeyValuePair<bool, string>?> DownloadCheckSums(string checkSumUrl)
{
var checksums = await _httpClientHelper.ReadString(checkSumUrl);
if ( checksums.Key )
{
return checksums;
}

_logger.LogError($"Checksum loading failed {CheckSumLocation}, next retry from mirror ~ error > " + checksums.Value);

checksums = await _httpClientHelper.ReadString(CheckSumLocationMirror);
if ( checksums.Key ) return new KeyValuePair<bool, string>(false, checksums.Value);

_logger.LogError($"Checksum loading failed {CheckSumLocationMirror}" +
$", next stop; please connect to internet and restart app ~ error > " + checksums.Value);
return null;
}

internal async Task<bool> StartDownloadForUnix()
{
var checksums = await DownloadCheckSums();
if ( checksums == null ) return false;
if ( checksums == null )
{
return false;
}
var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value);
return await DownloadForUnix(matchExifToolForUnixName,
GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key);
GetChecksumsFromTextFile(checksums.Value.Value));
}

internal static string GetUnixTarGzFromChecksum(string checksumsValue)
{
// (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip
var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()Image-ExifTool-[0-9\.]+\.tar.gz",
var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()Image-ExifTool-[0-9\.]+\.tar.gz",
RegexOptions.None, TimeSpan.FromMilliseconds(100));
return regexExifToolForWindowsName.Match(checksumsValue).Value;
}
Expand All @@ -150,26 +158,46 @@ private string ExeExifToolUnixFullFilePath()
return path;
}

internal async Task<bool> DownloadForUnix(string matchExifToolForUnixName,
string[] getChecksumsFromTextFile, bool downloadFromMirror = false)
internal async Task<bool> DownloadForUnix(string matchExifToolForUnixName, string[] getChecksumsFromTextFile)
{
var result = await DownloadForUnix(ExiftoolDownloadBasePath, matchExifToolForUnixName,
getChecksumsFromTextFile);

if ( result )
{
return true;
}

if ( _hostFileSystemStorage.ExistFile(
ExeExifToolUnixFullFilePath()) ) return true;
return await DownloadForUnix(ExiftoolDownloadBasePathMirror, matchExifToolForUnixName,
getChecksumsFromTextFile);
}


private async Task<bool> DownloadForUnix(string exiftoolDownloadBasePath, string matchExifToolForUnixName,
string[] getChecksumsFromTextFile)
{

if ( _hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) )
{
return true;
}

var tarGzArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.tar.gz");

var url = $"{ExiftoolDownloadBasePath}{matchExifToolForUnixName}";
if ( downloadFromMirror ) url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForUnixName}";
var url = $"{exiftoolDownloadBasePath}{matchExifToolForUnixName}";

var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath);
if ( !unixDownloaded )
{
throw new HttpRequestException($"file is not downloaded {matchExifToolForUnixName}");
_logger.LogError($"file is not downloaded {matchExifToolForUnixName}");
return false;
}
if ( !CheckSha1(tarGzArchiveFullFilePath, getChecksumsFromTextFile) )

if ( !CheckSha256(tarGzArchiveFullFilePath, getChecksumsFromTextFile) )
{
throw new HttpRequestException($"checksum for {tarGzArchiveFullFilePath} is not valid");
_logger.LogError($"Checksum for {tarGzArchiveFullFilePath} is not valid");
_hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath);
return false;
}

await new TarBal(_hostFileSystemStorage).ExtractTarGz(
Expand Down Expand Up @@ -229,13 +257,13 @@ internal async Task<bool> StartDownloadForWindows()

var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value);
return await DownloadForWindows(matchExifToolForWindowsName,
GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key);
GetChecksumsFromTextFile(checksums.Value.Value));
}

internal static string GetWindowsZipFromChecksum(string checksumsValue)
{
// (?<=SHA1\()exiftool-[\d\.]+\.zip
var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()exiftool-[0-9\.]+\.zip",
// (?<=SHA256\()exiftool-[\d\.]+_64\.zip
var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()exiftool-[0-9\.]+_64\.zip",
RegexOptions.None, TimeSpan.FromMilliseconds(100));
return regexExifToolForWindowsName.Match(checksumsValue).Value;
}
Expand All @@ -244,32 +272,36 @@ internal static string GetWindowsZipFromChecksum(string checksumsValue)
/// Parse the content of checksum file
/// </summary>
/// <param name="checksumsValue">input file: see test for example</param>
/// <param name="max">max number of SHA1 results</param>
/// <param name="max">max number of SHA256 results</param>
/// <returns></returns>
internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 8)
internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 20)
{
var regexExifToolForWindowsName = new Regex("[a-z0-9]{40}",
// SHA256 = 64 characters, SHA1 = 40 characters
var regexExifToolForWindowsName = new Regex("[a-z0-9]{64}",
RegexOptions.None, TimeSpan.FromMilliseconds(100));
var results = regexExifToolForWindowsName.Matches(checksumsValue).
Select(m => m.Value).
ToArray();
if ( results.Length < max ) return results;
if ( results.Length < max )
{
return results;
}

_logger.LogError($"More than {max} checksums found, this is not expected, code stops now");
return Array.Empty<string>();
return [];
}

/// <summary>
/// Check if SHA1 hash is valid
/// Instead of SHA1CryptoServiceProvider, we use SHA1.Create
/// Check if SHA256 hash is valid
/// Instead of SHA1CryptoServiceProvider, we use SHA256.Create
/// </summary>
/// <param name="fullFilePath">path of exiftool.exe</param>
/// <param name="checkSumOptions">list of sha1 hashes</param>
/// <param name="checkSumOptions">list of SHA256 hashes</param>
/// <returns></returns>
internal bool CheckSha1(string fullFilePath, IEnumerable<string> checkSumOptions)
internal bool CheckSha256(string fullFilePath, IEnumerable<string> checkSumOptions)
{
using var buffer = _hostFileSystemStorage.ReadStream(fullFilePath);
using var hashAlgorithm = SHA1.Create();
using var hashAlgorithm = SHA256.Create();

var byteHash = hashAlgorithm.ComputeHash(buffer);
var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty).ToLowerInvariant();
Expand All @@ -281,27 +313,44 @@ private string ExeExifToolWindowsFullFilePath()
return Path.Combine(Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"), "exiftool.exe");
}

internal async Task<bool> DownloadForWindows(string matchExifToolForWindowsName,
string[] getChecksumsFromTextFile, bool downloadFromMirror = false)
internal async Task<bool> DownloadForWindows(string matchExifToolForWindowsName, string[] getChecksumsFromTextFile)
{
var result = await DownloadForWindows(ExiftoolDownloadBasePath, matchExifToolForWindowsName,
getChecksumsFromTextFile);

if ( result )
{
return true;
}

return await DownloadForWindows(ExiftoolDownloadBasePathMirror, matchExifToolForWindowsName,
getChecksumsFromTextFile);
}

private async Task<bool> DownloadForWindows(string exiftoolDownloadBasePath, string matchExifToolForWindowsName,
string[] getChecksumsFromTextFile)
{
if ( _hostFileSystemStorage.ExistFile(
ExeExifToolWindowsFullFilePath()) ) return true;
ExeExifToolWindowsFullFilePath()) )
{
return true;
}

var zipArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip");
var windowsExifToolFolder = Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows");

var url = $"{ExiftoolDownloadBasePath}{matchExifToolForWindowsName}";
if ( downloadFromMirror ) url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForWindowsName}";

var url = $"{exiftoolDownloadBasePath}{matchExifToolForWindowsName}";
var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath);
if ( !windowsDownloaded )
{
throw new HttpRequestException($"file is not downloaded {matchExifToolForWindowsName}");
_logger.LogError($"file is not downloaded {matchExifToolForWindowsName}");
return false;
}

if ( !CheckSha1(zipArchiveFullFilePath, getChecksumsFromTextFile) )
if ( !CheckSha256(zipArchiveFullFilePath, getChecksumsFromTextFile) )
{
throw new HttpRequestException($"checksum for {zipArchiveFullFilePath} is not valid");
_logger.LogError($"Checksum for {zipArchiveFullFilePath} is not valid");
return false;
}

_hostFileSystemStorage.CreateDirectory(windowsExifToolFolder);
Expand Down
6 changes: 4 additions & 2 deletions starsky/starskytest/FakeCreateAn/CreateAnExifToolTarGz.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@ public static class CreateAnExifToolTarGz
"w0T7vkp19bidTXzafTjL2sfn8Et+h/8m9/yXevoL3P+2s3j+ueu6jR92/v/o/V/dqV/0onrSrehON6YH6krT2d9" +
"Q+e7kAAAAAAAAAAAAAAAAAAAAoJAXspDxGwAoAAA=";

public static readonly ImmutableArray<byte> Bytes = Base64Helper.TryParse(ImageExifToolTarGzUnix).ToImmutableArray();

public static readonly ImmutableArray<byte> Bytes = [..Base64Helper.TryParse(ImageExifToolTarGzUnix)];
public const string Sha1 = "b386a6849ed5f911085cc56f37d20f127162b21c";

public const string Sha256 = "31490b44bdef861a58328c5be576ba577a2f7cd15200246d20696c0fd6b33a5d";
}
}
5 changes: 5 additions & 0 deletions starsky/starskytest/FakeCreateAn/CreateAnExifToolWindows.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,5 +298,10 @@ public static class CreateAnExifToolWindows
/// File hash to check the content of the zip file
/// </summary>
public static readonly string Sha1 = "0da554d4cf5f4c15591da109ae070742ecfceb65";

/// <summary>
/// File hash to check the content of the zip file
/// </summary>
public static readonly string Sha256 = "8d85367eeddc2d84e2770eb7eda8de90a98ed6681966803ec8448310dfd53c5b";
}
}
Loading

0 comments on commit df12b47

Please sign in to comment.