Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bug fixes #218

Closed
wants to merge 1 commit into from
Closed

bug fixes #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 53 additions & 25 deletions src/ArtifactoryUploader/ArtifactoryUploader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
using LCT.APICommunications.Interfaces;
using LCT.APICommunications.Model;
using LCT.APICommunications.Model.AQL;
using LCT.ArtifactoryUploader.Model;
using LCT.Services.Interface;
using log4net;
using System;
Expand All @@ -28,17 +27,16 @@ public static class ArtfactoryUploader
private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi");
private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo");
public static IJFrogService jFrogService { get; set; }
public static IJFrogApiCommunication JFrogApiCommInstance { get; set; }

public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToArtifactory component, int timeout, DisplayPackagesInfo displayPackagesInfo)

public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToArtifactory component, int timeout)
{
Logger.Debug("Starting UploadPackageToArtifactory method");
string operationType = component.PackageType == PackageType.ClearedThirdParty
|| component.PackageType == PackageType.Development ? "copy" : "move";
string operationType = component.PackageType == PackageType.ClearedThirdParty || component.PackageType == PackageType.Development ? "copy" : "move";
string dryRunSuffix = component.DryRun ? " dry-run" : "";
HttpResponseMessage responsemessage = new HttpResponseMessage();
try
{
IJFrogApiCommunication jfrogApicommunication;

// Package Information
var packageInfo = await GetPackageInfoWithRetry(jFrogService, component);
Expand All @@ -50,11 +48,25 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
};
}

ArtifactoryCredentials repoCredentials = new ArtifactoryCredentials()
{
ApiKey = component.ApiKey,
Email = component.Email
};

// Initialize JFrog API communication based on Component Type
jfrogApicommunication = component.ComponentType?.ToUpperInvariant() switch
{
"MAVEN" => new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout),
"PYTHON" => new PythonJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout),
_ => new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout)
};

// Perform Copy or Move operation
responsemessage = component.PackageType switch
{
PackageType.ClearedThirdParty or PackageType.Development => await JFrogApiCommInstance.CopyFromRemoteRepo(component),
PackageType.Internal => await JFrogApiCommInstance.MoveFromRepo(component),
PackageType.ClearedThirdParty or PackageType.Development => await jfrogApicommunication.CopyFromRemoteRepo(component),
PackageType.Internal => await jfrogApicommunication.MoveFromRepo(component),
_ => new HttpResponseMessage(HttpStatusCode.NotFound)
};

Expand All @@ -65,7 +77,8 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
return responsemessage;
}

await PackageUploadHelper.JfrogFoundPackagesAsync(component, displayPackagesInfo, operationType, responsemessage, dryRunSuffix);
Logger.Info($"Successful{dryRunSuffix} {operationType} package {component.PackageName}-{component.Version}" +
$" from {component.SrcRepoName} to {component.DestRepoName}");

}
catch (HttpRequestException ex)
Expand All @@ -74,12 +87,6 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload;
return responsemessage;
}
catch (InvalidOperationException ex)
{
Logger.Error($"Error has occurred in UploadPackageToArtifactory--{ex}");
responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload;
return responsemessage;
}
finally
{
Logger.Debug("Ending UploadPackageToArtifactory method");
Expand Down Expand Up @@ -107,26 +114,47 @@ public static void SetConfigurationValues()

private static async Task<AqlResult> GetPackageInfoWithRetry(IJFrogService jFrogService, ComponentsToArtifactory component)
{
string srcRepoNameLower = component.SrcRepoName.ToLower();
string packageNameLower = component.JfrogPackageName.ToLower();
string pathLower = component.Path.ToLower();
async Task<AqlResult> TryGetPackageInfo(string srcRepo, string packageName, string path)
=> await jFrogService.GetPackageInfo(srcRepo, packageName, path);

var packageInfo = await jFrogService.GetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path);
var packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path);

if (component.ComponentType == "DEBIAN" && packageInfo.Name != component.JfrogPackageName)
// Handle DEBIAN package name mismatch
if (component.ComponentType == "DEBIAN" && packageInfo?.Name != component.JfrogPackageName)
{
component.CopyPackageApiUrl = component.CopyPackageApiUrl.Replace(component.JfrogPackageName, packageInfo.Name);
}

// Retry with lowercase values if packageInfo is still null
if (packageInfo == null)
{
// Retry with lowercase parameters
var lowercasePackageInfo = await jFrogService.GetPackageInfo(srcRepoNameLower, packageNameLower, pathLower);
var lowerSrcRepo = component.SrcRepoName.ToLower();
var lowerPackageName = component.JfrogPackageName.ToLower();
var lowerPath = component.Path.ToLower();

packageInfo = await TryGetPackageInfo(lowerSrcRepo, lowerPackageName, lowerPath);

if (lowercasePackageInfo != null)
if (packageInfo != null)
{
// Update the package API URL
component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower();
packageInfo = lowercasePackageInfo;
}
}

// Retry with wildcard path if still not found
// ToDo - A better way would need to be thought of in the future.
if (packageInfo == null)
{
packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, $"{component.Path}*");

if (packageInfo != null)
{
// Build URLs
string BuildUrl(string apiConstant) =>
$"{component.JfrogApi}{apiConstant}{component.SrcRepoName}/{packageInfo.Path}/{packageInfo.Name}" +
$"?to=/{component.DestRepoName}/{packageInfo.Path}/{packageInfo.Name}";

component.CopyPackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.CopyPackageApi)}&dry=1" : BuildUrl(ApiConstant.CopyPackageApi);
component.MovePackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.MovePackageApi)}&dry=1" : BuildUrl(ApiConstant.MovePackageApi);
}
}

Expand Down
81 changes: 23 additions & 58 deletions src/LCT.Common/CommonHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
// --------------------------------------------------------------------------------------------------------------------

using CycloneDX.Models;
using LCT.ArtifactPublisher;
using LCT.Common.Constants;
using LCT.Common.Model;
using log4net;
Expand All @@ -14,7 +13,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;

namespace LCT.Common
{
Expand All @@ -26,7 +24,6 @@ public static class CommonHelper
private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public static string ProjectSummaryLink { get; set; }

#region public
public static bool IsAzureDevOpsDebugEnabled()
{
string azureDevOpsDebug = System.Environment.GetEnvironmentVariable("System.Debug");
Expand All @@ -50,8 +47,8 @@ public static List<Component> RemoveExcludedComponents(List<Component> Component
{
name = $"{component.Group}/{component.Name}";
}
if (excludedcomponent.Length > 0 && (Regex.IsMatch(name.ToLowerInvariant(), WildcardToRegex(excludedcomponent[0].ToLowerInvariant()))) &&
(component.Version.ToLowerInvariant().Contains(excludedcomponent[1].ToLowerInvariant()) || excludedcomponent[1].ToLowerInvariant() == "*"))

if (name.ToLowerInvariant() == excludedcomponent[0].ToLowerInvariant() && excludedcomponent.Length > 0 && (component.Version.ToLowerInvariant() == excludedcomponent[1].ToLowerInvariant() || excludedcomponent[1].ToLowerInvariant() == "*"))
{
noOfExcludedComponents++;
ExcludedList.Add(component);
Expand All @@ -62,12 +59,26 @@ public static List<Component> RemoveExcludedComponents(List<Component> Component
return ComponentList;
}

public static List<Dependency> RemoveInvalidDependenciesAndReferences(List<Component> components, List<Dependency> dependencies)
{
var componentBomRefs = new HashSet<string>(components.Select(c => c.BomRef));

dependencies.RemoveAll(dep => !componentBomRefs.Contains(dep.Ref));

foreach (var dep in dependencies)
{
dep.Dependencies?.RemoveAll(refItem => !componentBomRefs.Contains(refItem.Ref));
}

return dependencies;
}

public static string GetSubstringOfLastOccurance(string value, string separator)
{
string result = string.IsNullOrWhiteSpace(value) ? string.Empty : value;
if (result.Contains(separator))
{
result = result[(result.LastIndexOf(separator) + separator.Length)..];
result = result?[(result.LastIndexOf(separator) + separator.Length)..];
}

return result;
Expand Down Expand Up @@ -148,6 +159,12 @@ public static void WriteToConsoleTable(Dictionary<string, int> printData, Dictio
}
}

private static string Sw360URL(string sw360Env, string releaseId)
{
string sw360URL = $"{sw360Env}{"/group/guest/components/-/component/release/detailRelease/"}{releaseId}";
return sw360URL;
}

public static void WriteComponentsWithoutDownloadURLToKpi(List<ComparisonBomData> componentInfo, List<Components> lstReleaseNotCreated, string sw360URL)
{
const string Name = "Name";
Expand All @@ -156,7 +173,6 @@ public static void WriteComponentsWithoutDownloadURLToKpi(List<ComparisonBomData
if (componentInfo.Count > 0 || lstReleaseNotCreated.Count > 0)
{
Logger.Logger.Log(null, Level.Alert, "Action Item required by the user:\n", null);
PublishFilesToArtifact();
Environment.ExitCode = 2;
}

Expand Down Expand Up @@ -203,7 +219,6 @@ public static void WriteComponentsNotLinkedListInConsole(List<Components> compon

if (components.Count > 0)
{
PublishFilesToArtifact();
Environment.ExitCode = 2;
Logger.Logger.Log(null, Level.Alert, "* Components Not linked to project :", null);
Logger.Logger.Log(null, Level.Alert, " Can be linked manually OR Check the Logs AND RE-Run", null);
Expand Down Expand Up @@ -242,56 +257,6 @@ public static void GetDetailsforManuallyAdded(List<Component> componentsForBOM,
}
}

public static string AddSpecificValuesToBOMFormat(Bom listOfComponentsToBom)
{
string guid = Guid.NewGuid().ToString();
listOfComponentsToBom.SerialNumber = $"urn:uuid:{guid}";
listOfComponentsToBom.Version = 1;
listOfComponentsToBom.Metadata.Timestamp = DateTime.UtcNow;
var formattedString = CycloneDX.Json.Serializer.Serialize(listOfComponentsToBom);

return formattedString;
}

public static void CallEnvironmentExit(int code)
{
if (code == -1)
{
Publish artifactPublisher = new Publish(Log4Net.CatoolLogPath, FileOperations.CatoolBomFilePath);
artifactPublisher.UploadLogs();
EnvironmentExit(code);
}
}

public static void EnvironmentExit(int exitCode)
{
Environment.Exit(exitCode);
}

public static void PublishFilesToArtifact()
{
Publish artifactPublisher = new Publish(Log4Net.CatoolLogPath, FileOperations.CatoolBomFilePath);
artifactPublisher.UploadLogs();
artifactPublisher.UploadBom();
}

#endregion

#region private
private static string WildcardToRegex(string wildcard)
{
return "^" + Regex.Escape(wildcard).Replace("\\*", ".*") + "$";
}

private static string Sw360URL(string sw360Env, string releaseId)
{
string sw360URL = $"{sw360Env}{"/group/guest/components/-/component/release/detailRelease/"}{releaseId}";
return sw360URL;
}
#endregion
}
}




20 changes: 12 additions & 8 deletions src/LCT.PackageIdentifier/AlpineProcesser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,28 @@
using LCT.PackageIdentifier.Model;
using LCT.Services.Interface;
using log4net;
using Newtonsoft.Json;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading.Tasks;
using System.Xml;

namespace LCT.PackageIdentifier
{
/// <summary>
/// The AlpineProcessor class
/// </summary>
public class AlpineProcessor : IParser
public class AlpineProcessor : IParser
{
static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private readonly ICycloneDXBomParser _cycloneDXBomParser;
public AlpineProcessor(ICycloneDXBomParser cycloneDXBomParser)
readonly CycloneDXBomParser cycloneDXBomParser;

public AlpineProcessor()
{
_cycloneDXBomParser = cycloneDXBomParser;
cycloneDXBomParser = new CycloneDXBomParser();
}

#region public method
Expand Down Expand Up @@ -61,28 +64,30 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension))
{
Bom templateDetails;
templateDetails = CycloneDXBomParser.ExtractSBOMDetailsFromTemplate(_cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath));
templateDetails = CycloneDXBomParser.ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath));
CycloneDXBomParser.CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType);
//Adding Template Component Details & MetaData
SbomTemplate.AddComponentDetails(bom.Components, templateDetails);
}

bom = RemoveExcludedComponents(appSettings, bom);
bom.Dependencies = bom.Dependencies?.GroupBy(x => new { x.Ref }).Select(y => y.First()).ToList();
return bom;
}

public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM)
{
List<Component> componentForBOM = cycloneDXBOM.Components.ToList();
List<Dependency> dependenciesForBOM = cycloneDXBOM.Dependencies.ToList();
int noOfExcludedComponents = 0;
if (appSettings.Alpine.ExcludedComponents != null)
{
componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Alpine.ExcludedComponents, ref noOfExcludedComponents);
dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM);
BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents;

}
cycloneDXBOM.Components = componentForBOM;
cycloneDXBOM.Dependencies = dependenciesForBOM;
return cycloneDXBOM;
}

Expand Down Expand Up @@ -120,7 +125,7 @@ public List<AlpinePackage> ParseCycloneDX(string filePath, List<Dependency> depe

private void ExtractDetailsForJson(string filePath, ref List<AlpinePackage> alpinePackages, List<Dependency> dependenciesForBOM)
{
Bom bom = _cycloneDXBomParser.ParseCycloneDXBom(filePath);
Bom bom = cycloneDXBomParser.ParseCycloneDXBom(filePath);
foreach (var componentsInfo in bom.Components)
{
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++;
Expand Down Expand Up @@ -187,7 +192,6 @@ private static List<Component> FormComponentReleaseExternalID(List<AlpinePackage
Purl = GetReleaseExternalId(prop.Name, prop.Version)
};
component.BomRef = $"{Dataconstant.PurlCheck()["ALPINE"]}{Dataconstant.ForwardSlash}{prop.Name}@{prop.Version}?{distro}";
component.Type=Component.Classification.Library;
Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.Discovered };
component.Properties = new List<Property> { identifierType };
listComponentForBOM.Add(component);
Expand Down
Loading
Loading