Skip to content

Commit

Permalink
bug fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@malavikakrishnan123
malavikakrishnan123 committed Dec 13, 2024
1 parent 198b69b commit 8af24bf
Show file tree
Hide file tree
Showing 9 changed files with 294 additions and 958 deletions.
78 changes: 53 additions & 25 deletions src/ArtifactoryUploader/ArtifactoryUploader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
using LCT.APICommunications.Interfaces;
using LCT.APICommunications.Model;
using LCT.APICommunications.Model.AQL;
using LCT.ArtifactoryUploader.Model;
using LCT.Services.Interface;
using log4net;
using System;
Expand All @@ -28,17 +27,16 @@ public static class ArtfactoryUploader
private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi");
private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo");
public static IJFrogService jFrogService { get; set; }
public static IJFrogApiCommunication JFrogApiCommInstance { get; set; }

public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToArtifactory component, int timeout, DisplayPackagesInfo displayPackagesInfo)

public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToArtifactory component, int timeout)
{
Logger.Debug("Starting UploadPackageToArtifactory method");
string operationType = component.PackageType == PackageType.ClearedThirdParty
|| component.PackageType == PackageType.Development ? "copy" : "move";
string operationType = component.PackageType == PackageType.ClearedThirdParty || component.PackageType == PackageType.Development ? "copy" : "move";
string dryRunSuffix = component.DryRun ? " dry-run" : "";
HttpResponseMessage responsemessage = new HttpResponseMessage();
try
{
IJFrogApiCommunication jfrogApicommunication;

// Package Information
var packageInfo = await GetPackageInfoWithRetry(jFrogService, component);
Expand All @@ -50,11 +48,25 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
};
}

ArtifactoryCredentials repoCredentials = new ArtifactoryCredentials()
{
ApiKey = component.ApiKey,
Email = component.Email
};

// Initialize JFrog API communication based on Component Type
jfrogApicommunication = component.ComponentType?.ToUpperInvariant() switch
{
"MAVEN" => new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout),
"PYTHON" => new PythonJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout),
_ => new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout)
};

// Perform Copy or Move operation
responsemessage = component.PackageType switch
{
PackageType.ClearedThirdParty or PackageType.Development => await JFrogApiCommInstance.CopyFromRemoteRepo(component),
PackageType.Internal => await JFrogApiCommInstance.MoveFromRepo(component),
PackageType.ClearedThirdParty or PackageType.Development => await jfrogApicommunication.CopyFromRemoteRepo(component),
PackageType.Internal => await jfrogApicommunication.MoveFromRepo(component),
_ => new HttpResponseMessage(HttpStatusCode.NotFound)
};

Expand All @@ -65,7 +77,8 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
return responsemessage;
}

await PackageUploadHelper.JfrogFoundPackagesAsync(component, displayPackagesInfo, operationType, responsemessage, dryRunSuffix);
Logger.Info($"Successful{dryRunSuffix} {operationType} package {component.PackageName}-{component.Version}" +
$" from {component.SrcRepoName} to {component.DestRepoName}");

}
catch (HttpRequestException ex)
Expand All @@ -74,12 +87,6 @@ public static async Task<HttpResponseMessage> UploadPackageToRepo(ComponentsToAr
responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload;
return responsemessage;
}
catch (InvalidOperationException ex)
{
Logger.Error($"Error has occurred in UploadPackageToArtifactory--{ex}");
responsemessage.ReasonPhrase = ApiConstant.ErrorInUpload;
return responsemessage;
}
finally
{
Logger.Debug("Ending UploadPackageToArtifactory method");
Expand Down Expand Up @@ -107,26 +114,47 @@ public static void SetConfigurationValues()

private static async Task<AqlResult> GetPackageInfoWithRetry(IJFrogService jFrogService, ComponentsToArtifactory component)
{
string srcRepoNameLower = component.SrcRepoName.ToLower();
string packageNameLower = component.JfrogPackageName.ToLower();
string pathLower = component.Path.ToLower();
async Task<AqlResult> TryGetPackageInfo(string srcRepo, string packageName, string path)
=> await jFrogService.GetPackageInfo(srcRepo, packageName, path);

var packageInfo = await jFrogService.GetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path);
var packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path);

if (component.ComponentType == "DEBIAN" && packageInfo.Name != component.JfrogPackageName)
// Handle DEBIAN package name mismatch
if (component.ComponentType == "DEBIAN" && packageInfo?.Name != component.JfrogPackageName)
{
component.CopyPackageApiUrl = component.CopyPackageApiUrl.Replace(component.JfrogPackageName, packageInfo.Name);
}

// Retry with lowercase values if packageInfo is still null
if (packageInfo == null)
{
// Retry with lowercase parameters
var lowercasePackageInfo = await jFrogService.GetPackageInfo(srcRepoNameLower, packageNameLower, pathLower);
var lowerSrcRepo = component.SrcRepoName.ToLower();
var lowerPackageName = component.JfrogPackageName.ToLower();
var lowerPath = component.Path.ToLower();

packageInfo = await TryGetPackageInfo(lowerSrcRepo, lowerPackageName, lowerPath);

if (lowercasePackageInfo != null)
if (packageInfo != null)
{
// Update the package API URL
component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower();
packageInfo = lowercasePackageInfo;
}
}

// Retry with wildcard path if still not found
// ToDo - A better way would need to be thought of in the future.
if (packageInfo == null)
{
packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, $"{component.Path}*");

if (packageInfo != null)
{
// Build URLs
string BuildUrl(string apiConstant) =>
$"{component.JfrogApi}{apiConstant}{component.SrcRepoName}/{packageInfo.Path}/{packageInfo.Name}" +
$"?to=/{component.DestRepoName}/{packageInfo.Path}/{packageInfo.Name}";

component.CopyPackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.CopyPackageApi)}&dry=1" : BuildUrl(ApiConstant.CopyPackageApi);
component.MovePackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.MovePackageApi)}&dry=1" : BuildUrl(ApiConstant.MovePackageApi);
}
}

Expand Down
81 changes: 23 additions & 58 deletions src/LCT.Common/CommonHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
// --------------------------------------------------------------------------------------------------------------------

using CycloneDX.Models;
using LCT.ArtifactPublisher;
using LCT.Common.Constants;
using LCT.Common.Model;
using log4net;
Expand All @@ -14,7 +13,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;

namespace LCT.Common
{
Expand All @@ -26,7 +24,6 @@ public static class CommonHelper
private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public static string ProjectSummaryLink { get; set; }

#region public
public static bool IsAzureDevOpsDebugEnabled()
{
string azureDevOpsDebug = System.Environment.GetEnvironmentVariable("System.Debug");
Expand All @@ -50,8 +47,8 @@ public static List<Component> RemoveExcludedComponents(List<Component> Component
{
name = $"{component.Group}/{component.Name}";
}
if (excludedcomponent.Length > 0 && (Regex.IsMatch(name.ToLowerInvariant(), WildcardToRegex(excludedcomponent[0].ToLowerInvariant()))) &&
(component.Version.ToLowerInvariant().Contains(excludedcomponent[1].ToLowerInvariant()) || excludedcomponent[1].ToLowerInvariant() == "*"))

if (name.ToLowerInvariant() == excludedcomponent[0].ToLowerInvariant() && excludedcomponent.Length > 0 && (component.Version.ToLowerInvariant() == excludedcomponent[1].ToLowerInvariant() || excludedcomponent[1].ToLowerInvariant() == "*"))
{
noOfExcludedComponents++;
ExcludedList.Add(component);
Expand All @@ -62,12 +59,26 @@ public static List<Component> RemoveExcludedComponents(List<Component> Component
return ComponentList;
}

public static List<Dependency> RemoveInvalidDependenciesAndReferences(List<Component> components, List<Dependency> dependencies)
{
var componentBomRefs = new HashSet<string>(components.Select(c => c.BomRef));

dependencies.RemoveAll(dep => !componentBomRefs.Contains(dep.Ref));

foreach (var dep in dependencies)
{
dep.Dependencies?.RemoveAll(refItem => !componentBomRefs.Contains(refItem.Ref));
}

return dependencies;
}

public static string GetSubstringOfLastOccurance(string value, string separator)
{
string result = string.IsNullOrWhiteSpace(value) ? string.Empty : value;
if (result.Contains(separator))
{
result = result[(result.LastIndexOf(separator) + separator.Length)..];
result = result?[(result.LastIndexOf(separator) + separator.Length)..];
}

return result;
Expand Down Expand Up @@ -148,6 +159,12 @@ public static void WriteToConsoleTable(Dictionary<string, int> printData, Dictio
}
}

private static string Sw360URL(string sw360Env, string releaseId)
{
string sw360URL = $"{sw360Env}{"/group/guest/components/-/component/release/detailRelease/"}{releaseId}";
return sw360URL;
}

public static void WriteComponentsWithoutDownloadURLToKpi(List<ComparisonBomData> componentInfo, List<Components> lstReleaseNotCreated, string sw360URL)
{
const string Name = "Name";
Expand All @@ -156,7 +173,6 @@ public static void WriteComponentsWithoutDownloadURLToKpi(List<ComparisonBomData
if (componentInfo.Count > 0 || lstReleaseNotCreated.Count > 0)
{
Logger.Logger.Log(null, Level.Alert, "Action Item required by the user:\n", null);
PublishFilesToArtifact();
Environment.ExitCode = 2;
}

Expand Down Expand Up @@ -203,7 +219,6 @@ public static void WriteComponentsNotLinkedListInConsole(List<Components> compon

if (components.Count > 0)
{
PublishFilesToArtifact();
Environment.ExitCode = 2;
Logger.Logger.Log(null, Level.Alert, "* Components Not linked to project :", null);
Logger.Logger.Log(null, Level.Alert, " Can be linked manually OR Check the Logs AND RE-Run", null);
Expand Down Expand Up @@ -242,56 +257,6 @@ public static void GetDetailsforManuallyAdded(List<Component> componentsForBOM,
}
}

public static string AddSpecificValuesToBOMFormat(Bom listOfComponentsToBom)
{
string guid = Guid.NewGuid().ToString();
listOfComponentsToBom.SerialNumber = $"urn:uuid:{guid}";
listOfComponentsToBom.Version = 1;
listOfComponentsToBom.Metadata.Timestamp = DateTime.UtcNow;
var formattedString = CycloneDX.Json.Serializer.Serialize(listOfComponentsToBom);

return formattedString;
}

public static void CallEnvironmentExit(int code)
{
if (code == -1)
{
Publish artifactPublisher = new Publish(Log4Net.CatoolLogPath, FileOperations.CatoolBomFilePath);
artifactPublisher.UploadLogs();
EnvironmentExit(code);
}
}

public static void EnvironmentExit(int exitCode)
{
Environment.Exit(exitCode);
}

public static void PublishFilesToArtifact()
{
Publish artifactPublisher = new Publish(Log4Net.CatoolLogPath, FileOperations.CatoolBomFilePath);
artifactPublisher.UploadLogs();
artifactPublisher.UploadBom();
}

#endregion

#region private
private static string WildcardToRegex(string wildcard)
{
return "^" + Regex.Escape(wildcard).Replace("\\*", ".*") + "$";
}

private static string Sw360URL(string sw360Env, string releaseId)
{
string sw360URL = $"{sw360Env}{"/group/guest/components/-/component/release/detailRelease/"}{releaseId}";
return sw360URL;
}
#endregion
}
}




20 changes: 12 additions & 8 deletions src/LCT.PackageIdentifier/AlpineProcesser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,28 @@
using LCT.PackageIdentifier.Model;
using LCT.Services.Interface;
using log4net;
using Newtonsoft.Json;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading.Tasks;
using System.Xml;

namespace LCT.PackageIdentifier
{
/// <summary>
/// The AlpineProcessor class
/// </summary>
public class AlpineProcessor : IParser
public class AlpineProcessor : IParser
{
static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private readonly ICycloneDXBomParser _cycloneDXBomParser;
public AlpineProcessor(ICycloneDXBomParser cycloneDXBomParser)
readonly CycloneDXBomParser cycloneDXBomParser;

public AlpineProcessor()
{
_cycloneDXBomParser = cycloneDXBomParser;
cycloneDXBomParser = new CycloneDXBomParser();
}

#region public method
Expand Down Expand Up @@ -61,28 +64,30 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension))
{
Bom templateDetails;
templateDetails = CycloneDXBomParser.ExtractSBOMDetailsFromTemplate(_cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath));
templateDetails = CycloneDXBomParser.ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath));
CycloneDXBomParser.CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType);
//Adding Template Component Details & MetaData
SbomTemplate.AddComponentDetails(bom.Components, templateDetails);
}

bom = RemoveExcludedComponents(appSettings, bom);
bom.Dependencies = bom.Dependencies?.GroupBy(x => new { x.Ref }).Select(y => y.First()).ToList();
return bom;
}

public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM)
{
List<Component> componentForBOM = cycloneDXBOM.Components.ToList();
List<Dependency> dependenciesForBOM = cycloneDXBOM.Dependencies.ToList();
int noOfExcludedComponents = 0;
if (appSettings.Alpine.ExcludedComponents != null)
{
componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Alpine.ExcludedComponents, ref noOfExcludedComponents);
dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM);
BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents;

}
cycloneDXBOM.Components = componentForBOM;
cycloneDXBOM.Dependencies = dependenciesForBOM;
return cycloneDXBOM;
}

Expand Down Expand Up @@ -120,7 +125,7 @@ public List<AlpinePackage> ParseCycloneDX(string filePath, List<Dependency> depe

private void ExtractDetailsForJson(string filePath, ref List<AlpinePackage> alpinePackages, List<Dependency> dependenciesForBOM)
{
Bom bom = _cycloneDXBomParser.ParseCycloneDXBom(filePath);
Bom bom = cycloneDXBomParser.ParseCycloneDXBom(filePath);
foreach (var componentsInfo in bom.Components)
{
BomCreator.bomKpiData.ComponentsinPackageLockJsonFile++;
Expand Down Expand Up @@ -187,7 +192,6 @@ private static List<Component> FormComponentReleaseExternalID(List<AlpinePackage
Purl = GetReleaseExternalId(prop.Name, prop.Version)
};
component.BomRef = $"{Dataconstant.PurlCheck()["ALPINE"]}{Dataconstant.ForwardSlash}{prop.Name}@{prop.Version}?{distro}";
component.Type=Component.Classification.Library;
Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.Discovered };
component.Properties = new List<Property> { identifierType };
listComponentForBOM.Add(component);
Expand Down
Loading

0 comments on commit 8af24bf

Please sign in to comment.