diff --git a/src/ArtifactoryUploader/ArtifactoryUploader.cs b/src/ArtifactoryUploader/ArtifactoryUploader.cs index 1ed33b5c..b2c19753 100644 --- a/src/ArtifactoryUploader/ArtifactoryUploader.cs +++ b/src/ArtifactoryUploader/ArtifactoryUploader.cs @@ -28,8 +28,8 @@ public static class ArtfactoryUploader private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi"); private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo"); public static IJFrogService jFrogService { get; set; } - public static IJFrogApiCommunication JFrogApiCommInstance { get; set; } - + public static IJFrogApiCommunication JFrogApiCommInstance { get; set; } + public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout, DisplayPackagesInfo displayPackagesInfo) { Logger.Debug("Starting UploadPackageToArtifactory method"); @@ -107,26 +107,47 @@ public static void SetConfigurationValues() private static async Task GetPackageInfoWithRetry(IJFrogService jFrogService, ComponentsToArtifactory component) { - string srcRepoNameLower = component.SrcRepoName.ToLower(); - string packageNameLower = component.JfrogPackageName.ToLower(); - string pathLower = component.Path.ToLower(); + async Task TryGetPackageInfo(string srcRepo, string packageName, string path) + => await jFrogService.GetPackageInfo(srcRepo, packageName, path); - var packageInfo = await jFrogService.GetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path); + var packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, component.Path); - if (component.ComponentType == "DEBIAN" && packageInfo.Name != component.JfrogPackageName) + // Handle DEBIAN package name mismatch + if (component.ComponentType == "DEBIAN" && packageInfo?.Name != component.JfrogPackageName) { component.CopyPackageApiUrl = component.CopyPackageApiUrl.Replace(component.JfrogPackageName, packageInfo.Name); } + + // Retry with lowercase values if packageInfo is still null if (packageInfo == null) { - // Retry with lowercase parameters - var lowercasePackageInfo = await jFrogService.GetPackageInfo(srcRepoNameLower, packageNameLower, pathLower); + var lowerSrcRepo = component.SrcRepoName.ToLower(); + var lowerPackageName = component.JfrogPackageName.ToLower(); + var lowerPath = component.Path.ToLower(); + + packageInfo = await TryGetPackageInfo(lowerSrcRepo, lowerPackageName, lowerPath); - if (lowercasePackageInfo != null) + if (packageInfo != null) { - // Update the package API URL component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower(); - packageInfo = lowercasePackageInfo; + } + } + + // Retry with wildcard path if still not found + // ToDo - A better way would need to be thought of in the future. + if (packageInfo == null) + { + packageInfo = await TryGetPackageInfo(component.SrcRepoName, component.JfrogPackageName, $"{component.Path}*"); + + if (packageInfo != null) + { + // Build URLs + string BuildUrl(string apiConstant) => + $"{component.JfrogApi}{apiConstant}{component.SrcRepoName}/{packageInfo.Path}/{packageInfo.Name}" + + $"?to=/{component.DestRepoName}/{packageInfo.Path}/{packageInfo.Name}"; + + component.CopyPackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.CopyPackageApi)}&dry=1" : BuildUrl(ApiConstant.CopyPackageApi); + component.MovePackageApiUrl = component.DryRun ? $"{BuildUrl(ApiConstant.MovePackageApi)}&dry=1" : BuildUrl(ApiConstant.MovePackageApi); } } diff --git a/src/LCT.Common/CommonHelper.cs b/src/LCT.Common/CommonHelper.cs index 257550e6..cd528f17 100644 --- a/src/LCT.Common/CommonHelper.cs +++ b/src/LCT.Common/CommonHelper.cs @@ -62,6 +62,20 @@ public static List RemoveExcludedComponents(List Component return ComponentList; } + public static List RemoveInvalidDependenciesAndReferences(List components, List dependencies) + { + var componentBomRefs = new HashSet(components.Select(c => c.BomRef)); + + dependencies.RemoveAll(dep => !componentBomRefs.Contains(dep.Ref)); + + foreach (var dep in dependencies) + { + dep.Dependencies?.RemoveAll(refItem => !componentBomRefs.Contains(refItem.Ref)); + } + + return dependencies; + } + public static string GetSubstringOfLastOccurance(string value, string separator) { string result = string.IsNullOrWhiteSpace(value) ? string.Empty : value; diff --git a/src/LCT.PackageIdentifier/AlpineProcesser.cs b/src/LCT.PackageIdentifier/AlpineProcesser.cs index f284432f..9607d14d 100644 --- a/src/LCT.PackageIdentifier/AlpineProcesser.cs +++ b/src/LCT.PackageIdentifier/AlpineProcesser.cs @@ -75,14 +75,17 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Alpine.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Alpine.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index 9bade284..936da4a4 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -524,13 +524,16 @@ private static void GetDistinctComponentList(ref List listofComponent private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Conan.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/DebianProcessor.cs b/src/LCT.PackageIdentifier/DebianProcessor.cs index 8a04641b..f4dccba7 100644 --- a/src/LCT.PackageIdentifier/DebianProcessor.cs +++ b/src/LCT.PackageIdentifier/DebianProcessor.cs @@ -110,13 +110,16 @@ private void AddSiemensDirectProperty(ref Bom bom) public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Debian.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Debian.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 3f840b93..511fdea7 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -100,6 +100,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) if (appSettings.Maven.ExcludedComponents != null) { componentsForBOM = CommonHelper.RemoveExcludedComponents(componentsForBOM, appSettings.Maven.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentsForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } diff --git a/src/LCT.PackageIdentifier/NpmProcessor.cs b/src/LCT.PackageIdentifier/NpmProcessor.cs index 4d9ef296..d269025b 100644 --- a/src/LCT.PackageIdentifier/NpmProcessor.cs +++ b/src/LCT.PackageIdentifier/NpmProcessor.cs @@ -453,14 +453,17 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Npm.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Npm.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 20686163..a3450ec0 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -390,14 +390,17 @@ private static bool IsInternalNugetComponent(List aqlResultList, Comp public static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Nuget.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Nuget.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; } diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index e9f8ecbf..2daadc33 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -300,14 +300,17 @@ private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) { List componentForBOM = cycloneDXBOM.Components.ToList(); + List dependenciesForBOM = cycloneDXBOM.Dependencies?.ToList() ?? new List(); int noOfExcludedComponents = 0; if (appSettings.Python.ExcludedComponents != null) { componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Python.ExcludedComponents, ref noOfExcludedComponents); + dependenciesForBOM = CommonHelper.RemoveInvalidDependenciesAndReferences(componentForBOM, dependenciesForBOM); BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; } cycloneDXBOM.Components = componentForBOM; + cycloneDXBOM.Dependencies = dependenciesForBOM; return cycloneDXBOM; }