From 81f41774853e8d03ae8d725f803ec19ed70be3a2 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Fri, 13 Dec 2024 18:36:21 +0530 Subject: [PATCH 1/5] Re-writting the GetPackageInfoWithRetry with better standards and also adding a wildcard search --- .../ArtifactoryUploader.cs | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/src/ArtifactoryUploader/ArtifactoryUploader.cs b/src/ArtifactoryUploader/ArtifactoryUploader.cs index 1ed33b5c..b1925877 100644 --- a/src/ArtifactoryUploader/ArtifactoryUploader.cs +++ b/src/ArtifactoryUploader/ArtifactoryUploader.cs @@ -28,12 +28,12 @@ public static class ArtfactoryUploader private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi"); private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo"); public static IJFrogService jFrogService { get; set; } - public static IJFrogApiCommunication JFrogApiCommInstance { get; set; } - + public static IJFrogApiCommunication JFrogApiCommInstance { get; set; } + public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout, DisplayPackagesInfo displayPackagesInfo) { Logger.Debug("Starting UploadPackageToArtifactory method"); - string operationType = component.PackageType == PackageType.ClearedThirdParty + string operationType = component.PackageType == PackageType.ClearedThirdParty || component.PackageType == PackageType.Development ? "copy" : "move"; string dryRunSuffix = component.DryRun ? " dry-run" : ""; HttpResponseMessage responsemessage = new HttpResponseMessage(); @@ -107,26 +107,47 @@ public static void SetConfigurationValues() private static async Task GetPackageInfoWithRetry(IJFrogService jFrogService, ComponentsToArtifactory component) { - string srcRepoNameLower = component.SrcRepoName.ToLower(); - string packageNameLower = component.JfrogPackageName.ToLower(); - string pathLower = component.Path.ToLower(); + async Task TryGetPackageInfo(string srcRepo, string packageName, string path) + => await jFrogService.GetPackageInfo(srcRepo, packageName, path); - var packageInfo = await jFrogService.GetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path); + var packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path); - if (component.ComponentType == "DEBIAN" && packageInfo.Name != component.JfrogPackageName) + // Handle DEBIAN package name mismatch + if (component.ComponentType == "DEBIAN" && packageInfo?.Name != component.JfrogPackageName) { component.CopyPackageApiUrl = component.CopyPackageApiUrl.Replace(component.JfrogPackageName, packageInfo.Name); } + + // Retry with lowercase values if packageInfo is still null if (packageInfo == null) { - // Retry with lowercase parameters - var lowercasePackageInfo = await jFrogService.GetPackageInfo(srcRepoNameLower, packageNameLower, pathLower); + var lowerSrcRepo = component.SrcRepoName.ToLower(); + var lowerPackageName = component.JfrogPackageName.ToLower(); + var lowerPath = component.Path.ToLower(); + + packageInfo = await TryGetPackageInfo(lowerSrcRepo, lowerPackageName, lowerPath); - if (lowercasePackageInfo != null) + if (packageInfo != null) { - // Update the package API URL component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower(); - packageInfo = lowercasePackageInfo; + } + } + + // Retry with wildcard path if still not found + // ToDo - A better way would need to be thought of in the future. + if (packageInfo == null) + { + packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, $"{component.Path}*"); + + if (packageInfo != null) + { + // Build URLs + string BuildUrl(string apiConstant) => + $"{component.JfrogApi}{apiConstant}{component.SrcRepoName}/{packageInfo.Path}/{packageInfo.Name}" + + $"?to=/{component.DestRepoName}/{packageInfo.Path}/{packageInfo.Name}"; + + component.CopyPackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.CopyPackageApi)}&dry=1" : BuildUrl(ApiConstant.CopyPackageApi); + component.MovePackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.MovePackageApi)}&dry=1" : BuildUrl(ApiConstant.MovePackageApi); } } From c3436a62d0f2703d3eb8303097e176fe7c792b9f Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Fri, 13 Dec 2024 18:36:49 +0530 Subject: [PATCH 2/5] Adding a helper method to remove Invalid Dependencies And References --- src/LCT.Common/CommonHelper.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/LCT.Common/CommonHelper.cs b/src/LCT.Common/CommonHelper.cs index 257550e6..cd528f17 100644 --- a/src/LCT.Common/CommonHelper.cs +++ b/src/LCT.Common/CommonHelper.cs @@ -62,6 +62,20 @@ public static List RemoveExcludedComponents(List Component return ComponentList; } + public static List RemoveInvalidDependenciesAndReferences(List components, List dependencies) + { + var componentBomRefs = new HashSet(components.Select(c => c.BomRef)); + + dependencies.RemoveAll(dep => !componentBomRefs.Contains(dep.Ref)); + + foreach (var dep in dependencies) + { + dep.Dependencies?.RemoveAll(refItem => !componentBomRefs.Contains(refItem.Ref)); + } + + return dependencies; + } + public static string GetSubstringOfLastOccurance(string value, string separator) { string result = string.IsNullOrWhiteSpace(value) ? string.Empty : value; From d0b8f62952e9e7ec74e980ae5be46c5315013e8d Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Fri, 13 Dec 2024 18:37:27 +0530 Subject: [PATCH 3/5] Removing invalid dependencies after the RemoveExcludedComponents for all the package types --- src/LCT.PackageIdentifier/AlpineProcesser.cs | 3 +++ src/LCT.PackageIdentifier/ConanProcessor.cs | 3 +++ src/LCT.PackageIdentifier/DebianProcessor.cs | 3 +++ src/LCT.PackageIdentifier/MavenProcessor.cs | 1 + src/LCT.PackageIdentifier/NpmProcessor.cs | 3 +++ src/LCT.PackageIdentifier/NugetProcessor.cs | 3 +++ src/LCT.PackageIdentifier/PythonProcessor.cs | 3 +++ 7 files changed, 19 insertions(+) diff --git a/src/LCT.PackageIdentifier/AlpineProcesser.cs b/src/LCT.PackageIdentifier/AlpineProcesser.cs index f284432f..c062b250 100644 --- a/src/LCT.PackageIdentifier/AlpineProcesser.cs +++ b/src/LCT.PackageIdentifier/AlpineProcesser.cs @@ -75,14 +75,17 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Alpine.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Alpine.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index 9bade284..c723b85e 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -524,13 +524,16 @@ private static void GetDistinctComponentList(ref List listofComponent private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Conan.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index 8a04641b..9dbb441a 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -110,13 +110,16 @@ private void AddSiemensDirectProperty(ref Bom bom) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Debian.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Debian.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 3f840b93..511fdea7 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -100,6 +100,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) if (appSettings.Maven.ExcludedComponents != null) { componentsForBOM = CommonHelper.RemoveExcludedComponents(componentsForBOM, appSettings.Maven.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentsForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 4d9ef296..a850b568 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -453,14 +453,17 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Npm.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Npm.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 20686163..5a0fd9b4 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -390,14 +390,17 @@ private static bool IsInternalNugetComponent(List aqlResultList, Comp public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Nuget.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Nuget.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index e9f8ecbf..50c79e3c 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -300,14 +300,17 @@ private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); int noOfExcludedComponents = 0; if (appSettings.Python.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Python.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } From b93b961f5e8fceaa16557ec92d0bed515b3e4f66 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Fri, 13 Dec 2024 18:46:27 +0530 Subject: [PATCH 4/5] whitespace changes --- src/ArtifactoryUploader/ArtifactoryUploader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ArtifactoryUploader/ArtifactoryUploader.cs b/src/ArtifactoryUploader/ArtifactoryUploader.cs index b1925877..b2c19753 100644 --- a/src/ArtifactoryUploader/ArtifactoryUploader.cs +++ b/src/ArtifactoryUploader/ArtifactoryUploader.cs @@ -33,7 +33,7 @@ public static class ArtfactoryUploader public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout, DisplayPackagesInfo displayPackagesInfo) { Logger.Debug("Starting UploadPackageToArtifactory method"); - string operationType = component.PackageType == PackageType.ClearedThirdParty + string operationType = component.PackageType == PackageType.ClearedThirdParty || component.PackageType == PackageType.Development ? "copy" : "move"; string dryRunSuffix = component.DryRun ? " dry-run" : ""; HttpResponseMessage responsemessage = new HttpResponseMessage(); From b76a2ae2e1c94d96b039640c0d698f297d90af94 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Fri, 13 Dec 2024 19:03:42 +0530 Subject: [PATCH 5/5] Adding an empty list for dependecy if its not found --- src/LCT.PackageIdentifier/AlpineProcesser.cs | 2 +- src/LCT.PackageIdentifier/ConanProcessor.cs | 2 +- src/LCT.PackageIdentifier/DebianProcessor.cs | 2 +- src/LCT.PackageIdentifier/NpmProcessor.cs | 2 +- src/LCT.PackageIdentifier/NugetProcessor.cs | 2 +- src/LCT.PackageIdentifier/PythonProcessor.cs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/LCT.PackageIdentifier/AlpineProcesser.cs b/src/LCT.PackageIdentifier/AlpineProcesser.cs index c062b250..9607d14d 100644 --- a/src/LCT.PackageIdentifier/AlpineProcesser.cs +++ b/src/LCT.PackageIdentifier/AlpineProcesser.cs @@ -75,7 +75,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Alpine.ExcludedComponents != null) { diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index c723b85e..936da4a4 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -524,7 +524,7 @@ private static void GetDistinctComponentList(ref List listofComponent private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Conan.ExcludedComponents != null) { diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index 9dbb441a..f4dccba7 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -110,7 +110,7 @@ private void AddSiemensDirectProperty(ref Bom bom) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Debian.ExcludedComponents != null) { diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index a850b568..d269025b 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -453,7 +453,7 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Npm.ExcludedComponents != null) { diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 5a0fd9b4..a3450ec0 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -390,7 +390,7 @@ private static bool IsInternalNugetComponent(List aqlResultList, Comp public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Nuget.ExcludedComponents != null) { diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index 50c79e3c..2daadc33 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -300,7 +300,7 @@ private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); - List dependenciesForBOM = cycloneDXBOM.Dependencies.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Python.ExcludedComponents != null) {