From 9ef66c0bb282455ffc2b6f49d2776baddb7b9dd2 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Tue, 5 Sep 2023 17:47:46 +0530 Subject: [PATCH 01/18] Initial --- src/LCT.Common/CommonAppSettings.cs | 2 +- src/LCT.PackageIdentifier/Program.cs | 1 - src/LCT.PackageIdentifier/PythonProcessor.cs | 10 +- .../ComponentCreator.cs | 14 +-- src/LCT.SW360PackageCreator/CreatorHelper.cs | 57 ++++++--- .../DebianPackageDownloader.cs | 36 +----- .../Model/PythonPackage.cs | 18 +++ src/LCT.SW360PackageCreator/Program.cs | 10 +- src/LCT.SW360PackageCreator/URLHelper.cs | 119 +++++++++++++++++- 9 files changed, 190 insertions(+), 77 deletions(-) create mode 100644 src/LCT.SW360PackageCreator/Model/PythonPackage.cs diff --git a/src/LCT.Common/CommonAppSettings.cs b/src/LCT.Common/CommonAppSettings.cs index 7acb4cb1..5029c334 100644 --- a/src/LCT.Common/CommonAppSettings.cs +++ b/src/LCT.Common/CommonAppSettings.cs @@ -25,9 +25,9 @@ public class CommonAppSettings public static string PackageUrlApi { get; set; } = $"https://www.nuget.org/api/v2/package/"; public static string SourceURLNugetApi { get; set; } = $"https://api.nuget.org/v3-flatcontainer/"; public static string SourceURLMavenApi { get; set; } = $"https://repo.maven.apache.org/maven2/"; - public static string SnapshotBaseURL { get; set; } = $"https://snapshot.debian.org/mr/"; public static string SnapshotDownloadURL { get; set; } = $"https://snapshot.debian.org/archive/"; + public static string PyPiURL { get; set; } = $"https://pypi.org/pypi/"; public CommonAppSettings() { diff --git a/src/LCT.PackageIdentifier/Program.cs b/src/LCT.PackageIdentifier/Program.cs index 180d627c..d5cebcd2 100644 --- a/src/LCT.PackageIdentifier/Program.cs +++ b/src/LCT.PackageIdentifier/Program.cs @@ -69,7 +69,6 @@ static async Task Main(string[] args) $"SW360ProjectName\t --> {appSettings.SW360ProjectName}\n\t" + $"SW360ProjectID\t\t --> {appSettings.SW360ProjectID}\n\t" + $"ProjectType\t\t --> {appSettings.ProjectType}\n\t" + - $"RemoveDevDependency\t --> {appSettings.RemoveDevDependency}\n\t" + $"LogFolderPath\t\t --> {Path.GetFullPath(FolderPath)}", null); if (appSettings.IsTestMode) diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index 0b2be0ba..3058ba47 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -139,7 +139,7 @@ private static string GetReleaseExternalId(string name, string version) version = WebUtility.UrlEncode(version); version = version.Replace("%3A", ":"); - return $"{Dataconstant.PurlCheck()["PYTHON"]}{Dataconstant.ForwardSlash}{name}@{version}?arch=source"; + return $"{Dataconstant.PurlCheck()["PYTHON"]}{Dataconstant.ForwardSlash}{name}@{version}"; } private static List FormComponentReleaseExternalID(List listOfComponents) @@ -441,7 +441,7 @@ public async Task IdentificationOfInternalComponents(Co private static bool IsInternalPythonComponent(List aqlResultList, Component component, IBomHelper bomHelper) { - string jfrogcomponentName = $"{component.Name}-{component.Version}.tar.gz"; + string jfrogcomponentName = $"{component.Name}-{component.Version}{FileConstant.TargzFileExtension}"; if (aqlResultList.Exists(x => x.Name.Equals(jfrogcomponentName, StringComparison.OrdinalIgnoreCase))) { return true; @@ -451,7 +451,7 @@ private static bool IsInternalPythonComponent(List aqlResultList, Com string fullNameVersion = $"{fullName}-{component.Version}"; if (!fullNameVersion.Equals(jfrogcomponentName, StringComparison.OrdinalIgnoreCase) && aqlResultList.Exists( - x => x.Name.Equals(fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(".tar.gz")))) + x => x.Name.Equals(fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(FileConstant.TargzFileExtension)))) { return true; } @@ -486,7 +486,7 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List aqlResultList, Component component, IBomHelper bomHelper) { - string jfrogcomponentName = $"{component.Name}-{component.Version}.tar.gz"; + string jfrogcomponentName = $"{component.Name}-{component.Version}{FileConstant.TargzFileExtension}"; string repoName = aqlResultList.Find(x => x.Name.Equals( jfrogcomponentName, StringComparison.OrdinalIgnoreCase))?.Repo ?? NotFoundInRepo; @@ -498,7 +498,7 @@ private static string GetArtifactoryRepoName(List aqlResultList, Comp repoName.Equals(NotFoundInRepo, StringComparison.OrdinalIgnoreCase)) { repoName = aqlResultList.Find(x => x.Name.Contains( - fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(".tar.gz")))?.Repo ?? NotFoundInRepo; + fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(FileConstant.TargzFileExtension)))?.Repo ?? NotFoundInRepo; } return repoName; diff --git a/src/LCT.SW360PackageCreator/ComponentCreator.cs b/src/LCT.SW360PackageCreator/ComponentCreator.cs index 00d1071b..f3156c3f 100644 --- a/src/LCT.SW360PackageCreator/ComponentCreator.cs +++ b/src/LCT.SW360PackageCreator/ComponentCreator.cs @@ -45,7 +45,7 @@ public async Task> CycloneDxBomParser(CommonAppSettings { bom = cycloneDXBomParser.ParseCycloneDXBom(appSettings.BomFilePath); TotalComponentsFromPackageIdentifier = bom != null ? bom.Components.Count : 0; - ListofBomComponents = await GetListOfBomData(bom?.Components ?? new List(),appSettings); + ListofBomComponents = await GetListOfBomData(bom?.Components ?? new List(), appSettings); // Removing Duplicates ListofBomComponents = RemoveDuplicateComponents(ListofBomComponents); @@ -71,7 +71,7 @@ private async Task> GetListOfBomData(List components { Logger.Debug($"{item.Name}-{item.Version} found as internal component. "); } - else if(componentsData.IsDev=="true" && appSettings.RemoveDevDependency) + else if (componentsData.IsDev == "true" && appSettings.RemoveDevDependency) { //do nothing } @@ -191,6 +191,9 @@ private static async Task GetSourceUrl(string name, string version, componentsData = debComponentData; componentsData.ProjectType = projectType; break; + case "PYTHON": + componentsData.SourceUrl = await UrlHelper.Instance.GetSourceUrlForPythonPackage(name, version); + break; default: break; } @@ -252,10 +255,8 @@ private async Task CreateComponent(ICreatorHelper creatorHelper, try { - foreach (ComparisonBomData item in componentsToBoms) { - await CreateComponentAndRealease(creatorHelper, sw360CreatorService, item, sw360Url, appSettings); } } @@ -289,8 +290,6 @@ private async Task CreateComponentAndRealease(ICreatorHelper creatorHelper, private async Task CreateComponentAndReleaseWhenNotAvailable(ComparisonBomData item, ISw360CreatorService sw360CreatorService, ICreatorHelper creatorHelper, CommonAppSettings appSettings) { - - if (item.ComponentStatus == Dataconstant.NotAvailable && item.ReleaseStatus == Dataconstant.NotAvailable) { Logger.Logger.Log(null, Level.Notice, $"Creating the Component & Release : Name - {item.Name} , version - {item.Version}", null); @@ -303,7 +302,6 @@ private async Task CreateComponentAndReleaseWhenNotAvailable(ComparisonBomData i return; } - //till here ComponentCreateStatus createdStatus = await sw360CreatorService.CreateComponentBasesOFswComaprisonBOM(item, attachmentUrlList); @@ -319,8 +317,6 @@ private async Task CreateComponentAndReleaseWhenNotAvailable(ComparisonBomData i { await TriggeringFossologyUploadAndUpdateAdditionalData(item, sw360CreatorService, appSettings); } - - UpdatedCompareBomData.Add(item); } } diff --git a/src/LCT.SW360PackageCreator/CreatorHelper.cs b/src/LCT.SW360PackageCreator/CreatorHelper.cs index 665ddcae..8badf1d2 100644 --- a/src/LCT.SW360PackageCreator/CreatorHelper.cs +++ b/src/LCT.SW360PackageCreator/CreatorHelper.cs @@ -71,17 +71,11 @@ public async Task> DownloadReleaseAttachmentSource(Co { ServicePointManager.Expect100Continue = true; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; - - - Logger.Debug($"DownloadReleaseAttachmentSource()-Name-{component.Name},version-{component.Version},localPathforDownload-{localPathforDownload}"); - if (string.IsNullOrEmpty(component.SourceUrl) || component.SourceUrl.Equals(Dataconstant.SourceUrlNotFound)) Logger.Warn($"Source URL is not Found for {component.Name}-{component.Version}"); - - if (component.DownloadUrl.Equals(Dataconstant.DownloadUrlNotFound)) { Logger.Warn($"Source file is not attached,Release source Download Url is not Found for {component.Name}-{component.Version}"); @@ -94,7 +88,6 @@ public async Task> DownloadReleaseAttachmentSource(Co } else { - await DownloadDependencyList(component); GetAttachmentUrlListForMvn(localPathforDownload, component, ref AttachmentUrlList); @@ -112,6 +105,10 @@ private async Task GetAttachmentUrlList(ComparisonBomData component, Dic downloadPath = await _packageDownloderList["DEBIAN"].DownloadPackage(component, localPathforDownload); } } + else if (component.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"])) + { + downloadPath = await GetAttachmentUrlListForPython(component, localPathforDownload); + } else { downloadPath = await _packageDownloderList["NPM"].DownloadPackage(component, localPathforDownload); @@ -152,9 +149,7 @@ private static async Task DownloadDependencyList(ComparisonBomData component) await processResult; } - - - private static void GetAttachmentUrlListForMvn(string localPathforDownload, ComparisonBomData component, ref Dictionary attachmentUrlList) + private static void GetAttachmentUrlListForMvn(string localPathforDownload, ComparisonBomData component, ref Dictionary attachmentUrlList) { localPathforDownload = $"{localPathforDownload}{component.Name}-{component.Version}-sources.jar"; @@ -165,6 +160,33 @@ private static void GetAttachmentUrlListForMvn(string localPathforDownload, Com } + private static async Task GetAttachmentUrlListForPython(ComparisonBomData component, string localPathforDownload) + { + string downloadPath = string.Empty; + try + { + string componenetFullName = UrlHelper.GetCorrectFileExtension(component.SourceUrl); + string downloadFilePath = $"{localPathforDownload}{componenetFullName}"; + Directory.CreateDirectory(Path.GetDirectoryName(downloadFilePath)); + + if (!string.IsNullOrEmpty(component.SourceUrl) && !component.SourceUrl.Equals(Dataconstant.SourceUrlNotFound)) + { + Uri uri = new Uri(component.SourceUrl); + downloadPath = await UrlHelper.DownloadFileAsync(uri, downloadFilePath); + } + } + catch (WebException ex) + { + Logger.Debug($"GetAttachmentUrlListForPython :WebException :Release Name : {component.Name}@{component.Version}-PackageUrl: ,Error {ex}"); + } + catch (UriFormatException ex) + { + Logger.Debug($"GetAttachmentUrlListForPython:Release Name : {component.Name}@{component.Version}: Error {ex}"); + } + + return downloadPath; + } + public async Task> SetContentsForComparisonBOM(List lstComponentForBOM, ISW360Service sw360Service) { Logger.Debug($"SetContentsForComparisonBOM():Start"); @@ -202,6 +224,10 @@ public async Task> SetContentsForComparisonBOM(List> SetContentsForComparisonBOM(List GetUpdatedComponentsDetails(List ListofBomCom new Property { Name = Dataconstant.Cdx_FossologyUrl, Value = comBom.FossologyLink } }; - if (!(bom.Components.Any(x => x.BomRef.Contains(Dataconstant.PurlCheck()["MAVEN"])))) + if (!bom.Components.Exists(x => x.BomRef.Contains(Dataconstant.PurlCheck()["MAVEN"]))) { - bom.Components.FirstOrDefault(com => string.IsNullOrEmpty(com.Group) ? com.Name == comBom.Name && com.Version.Contains(comBom.Version) + bom.Components.Find(com => string.IsNullOrEmpty(com.Group) ? com.Name == comBom.Name && com.Version.Contains(comBom.Version) : $"{com.Group}{Dataconstant.ForwardSlash}{com.Name}" == comBom.Name && com.Version.Contains(comBom.Version))?.Properties.AddRange(prop); } else { - bom.Components.FirstOrDefault(com => com.Name == comBom.Name && com.Version.Contains(comBom.Version))?.Properties.AddRange(prop); - + bom.Components.Find(com => com.Name == comBom.Name && com.Version.Contains(comBom.Version))?.Properties.AddRange(prop); } } catch (JsonSerializationException ex) { Logger.Debug($"GetUpdatedComponentsDetails() For Component = {comBom.Name} : {ex}"); } - } return bom; } diff --git a/src/LCT.SW360PackageCreator/DebianPackageDownloader.cs b/src/LCT.SW360PackageCreator/DebianPackageDownloader.cs index 27a2d7be..8a97d02d 100644 --- a/src/LCT.SW360PackageCreator/DebianPackageDownloader.cs +++ b/src/LCT.SW360PackageCreator/DebianPackageDownloader.cs @@ -148,7 +148,7 @@ private static async Task DownloadTarFileAndGetPath(ComparisonBomData co if (!string.IsNullOrEmpty(SourceUrl) && !component.SourceUrl.Equals(Dataconstant.SourceUrlNotFound)) { Uri uri = new Uri(SourceUrl); - downloadPath = await DownloadFileFromSnapshotorgAsync(uri, downloadFilePath); + downloadPath = await UrlHelper.DownloadFileAsync(uri, downloadFilePath); } } catch (WebException ex) @@ -163,40 +163,6 @@ private static async Task DownloadTarFileAndGetPath(ComparisonBomData co return downloadPath; } - private static async Task DownloadFileFromSnapshotorgAsync(Uri uri, string downloadFilePath) - { - string downloadedPath = string.Empty; - try - { - using (WebClient webClient = new WebClient()) - { - await webClient.DownloadFileTaskAsync(uri, downloadFilePath); - } - downloadedPath = downloadFilePath; - Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)} ,Downloaded Successfully!!"); - } - catch (WebException webex) - { - Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Error {webex}"); - //Waiting for server to up.. - await Task.Delay(4000); - try - { - using (WebClient webClient = new WebClient()) - { - await webClient.DownloadFileTaskAsync(uri, downloadFilePath); - } - downloadedPath = downloadFilePath; - Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Success in retry!!"); - } - catch (WebException) - { - Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Error in retry!!"); - } - } - return downloadedPath; - } - private string ApplyPatchforComponents(ComparisonBomData component, string localDownloadPath, string fileName) { Result result; diff --git a/src/LCT.SW360PackageCreator/Model/PythonPackage.cs b/src/LCT.SW360PackageCreator/Model/PythonPackage.cs new file mode 100644 index 00000000..18ea999a --- /dev/null +++ b/src/LCT.SW360PackageCreator/Model/PythonPackage.cs @@ -0,0 +1,18 @@ +using System.Diagnostics.CodeAnalysis; + +namespace LCT.SW360PackageCreator.Model +{ + [ExcludeFromCodeCoverage] + internal class PythonPackage + { + public string Name { get; set; } + + public string Version { get; set; } + + public string PurlID { get; set; } + + public string SourceUrl { get; set; } + + public string WheelUrl { get; set; } + } +} diff --git a/src/LCT.SW360PackageCreator/Program.cs b/src/LCT.SW360PackageCreator/Program.cs index 536e8f84..81fe28be 100644 --- a/src/LCT.SW360PackageCreator/Program.cs +++ b/src/LCT.SW360PackageCreator/Program.cs @@ -103,10 +103,12 @@ private static async Task InitiatePackageCreatorProcess(CommonAppSettings appSet ICycloneDXBomParser cycloneDXBomParser = new CycloneDXBomParser(); IDebianPatcher debianPatcher = new DebianPatcher(); - IDictionary _packageDownloderList = new Dictionary(); - _packageDownloderList.Add("NPM", new PackageDownloader()); - _packageDownloderList.Add("NUGET", new PackageDownloader()); - _packageDownloderList.Add("DEBIAN", new DebianPackageDownloader(debianPatcher)); + IDictionary _packageDownloderList = new Dictionary + { + { "NPM", new PackageDownloader() }, + { "NUGET", new PackageDownloader() }, + { "DEBIAN", new DebianPackageDownloader(debianPatcher) } + }; ICreatorHelper creatorHelper = new CreatorHelper(_packageDownloderList); diff --git a/src/LCT.SW360PackageCreator/URLHelper.cs b/src/LCT.SW360PackageCreator/URLHelper.cs index 9096db39..89dbc62d 100644 --- a/src/LCT.SW360PackageCreator/URLHelper.cs +++ b/src/LCT.SW360PackageCreator/URLHelper.cs @@ -92,8 +92,6 @@ public async Task GetSourceUrlForNugetPackage(string componentName, stri return sourceURL; } - - /// /// Gets the Source URL for NPM Package /// @@ -378,7 +376,7 @@ private static DebianPackage GetProperSourceURL(DebianPackage sourceURLDetails, } // Finding upstream source file(.orig.tar.*) - string OrigFile = URLs.FirstOrDefault(item => item.Contains(FileConstant.OrigTarFileExtension) && + string OrigFile = URLs.Find(item => item.Contains(FileConstant.OrigTarFileExtension) && (item.EndsWith(FileConstant.TargzFileExtension) || item.EndsWith(FileConstant.XzFileExtension) || item.EndsWith(FileConstant.TgzFileExtension) || @@ -392,7 +390,7 @@ private static DebianPackage GetProperSourceURL(DebianPackage sourceURLDetails, else { // Finding actual source file () - var sourceURL = URLs.FirstOrDefault(item => !item.Contains(FileConstant.OrigTarFileExtension) && !item.Contains(FileConstant.DSCFileExtension) + var sourceURL = URLs.Find(item => !item.Contains(FileConstant.OrigTarFileExtension) && !item.Contains(FileConstant.DSCFileExtension) && !item.Contains(FileConstant.DebianTarFileExtension) && !item.Contains(FileConstant.DebianFileExtension) && (item.EndsWith(FileConstant.TargzFileExtension) || item.EndsWith(FileConstant.XzFileExtension) @@ -406,7 +404,7 @@ private static DebianPackage GetProperSourceURL(DebianPackage sourceURLDetails, else { // Not able to distinguish source file, so taking DSC file as source file - var dscFile = URLs.FirstOrDefault(item => item.Contains(FileConstant.DSCFileExtension)); + var dscFile = URLs.Find(item => item.Contains(FileConstant.DSCFileExtension)); if (!string.IsNullOrEmpty(dscFile)) { @@ -442,6 +440,117 @@ private async Task RetryToGetSourceURlDetailsAsync(string name, s return sourceDetails; } + public async Task GetSourceUrlForPythonPackage(string componentName, string componenVersion) + { + Logger.Debug($"URLHelper.GetSourceUrlForPythonPackage():Start"); + string name = componentName.ToLowerInvariant(); + string version = componenVersion.ToLowerInvariant(); + var response = await GetResponseFromPyPiOrg(name, version); + string sourceURL = GetSourceURLFromPyPiResponse(response); + return sourceURL; + } + + private async Task GetResponseFromPyPiOrg(string componentName, string componenVersion) + { + string URL; + string result = ""; + try + { + URL = $"{CommonAppSettings.PyPiURL}{componentName}" + + $"{Dataconstant.ForwardSlash}{componenVersion}{Dataconstant.ForwardSlash}json"; + + var response = await httpClient.GetStringAsync(URL); + return response.ToString(); + } + catch (HttpRequestException ex) + { + Logger.Debug($"GetResponseFromPyPiOrg():HttpRequestException", ex); + } + catch (TaskCanceledException ex) + { + Logger.Debug($"GetResponseFromPyPiOrg():TaskCanceledException", ex); + } + return result; + } + + private string GetSourceURLFromPyPiResponse(string response) + { + string SourceURL = ""; + + try + { + JObject data = JObject.Parse(response); + JToken fileinformations = data["urls"]; + + foreach (JToken fileinfo in fileinformations.Children()) + { + var url = fileinfo["url"]; + + if (!string.IsNullOrEmpty(url.ToString()) && url.ToString().EndsWith(FileConstant.TargzFileExtension)) + { + SourceURL = url.ToString(); + } + } + } + catch (JsonReaderException ex) + { + Logger.Debug($"GetSourceURLFromPyPiResponse():", ex); + } + catch (IOException ex) + { + Logger.Debug($"GetSourceURLFromPyPiResponse():", ex); + } + return SourceURL; + } + + public static async Task DownloadFileAsync(Uri uri, string downloadFilePath) + { + string downloadedPath = string.Empty; + try + { + using (WebClient webClient = new WebClient()) + { + await webClient.DownloadFileTaskAsync(uri, downloadFilePath); + } + downloadedPath = downloadFilePath; + Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)} ,Downloaded Successfully!!"); + } + catch (WebException webex) + { + Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Error {webex}"); + //Waiting for server to up.. + await Task.Delay(4000); + try + { + using (WebClient webClient = new WebClient()) + { + await webClient.DownloadFileTaskAsync(uri, downloadFilePath); + } + downloadedPath = downloadFilePath; + Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Success in retry!!"); + } + catch (WebException) + { + Logger.Debug($"DownloadFileFromSnapshotorgAsync:File Name : {Path.GetFileName(downloadFilePath)},Error in retry!!"); + } + } + return downloadedPath; + } + + public static string GetCorrectFileExtension(string sourceURL) + { + int idx = sourceURL.LastIndexOf(Dataconstant.ForwardSlash); + string fullname = string.Empty; + + if (idx != -1) + { + fullname = sourceURL.Substring(idx + 1); + } + + return fullname; + } + + public void Dispose() { Dispose(true); From a15362974f9256eeff1529f9e01253e43ce4064e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karthika=20Geethanand=20=20=C2=AF=5C=5F=28=E3=83=84=29=5F/?= =?UTF-8?q?=C2=AF?= <40568919+karthika-g@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:48:32 +0530 Subject: [PATCH 02/18] Smoke Test Bug fixes (#86) * initial docs * latest sbom * Property addition in Dataconstant file and appsettings updation * Import multiple SBOM files from customer maven ,nuget,npm * CI by vg_dev and internal property addition in package identifier output_15062023 * SBOM creation * SBOM parsing For nuget and Maven * identifier type added * sbom_import debian * New pattern added for input filetype * Review comment implementation * Build failure fix * Multiple Sbom reading changes for NPM,DEBIAN * Unique Package reading for NPM * update * Check in for bom reading * UT & IT changes * maven cycloneDx parsing logic * ununsed method removal * PR Review changes * Update CA_UsageDocument.md * Update CA-Tool--SBOM-Vision.md * Npm Change * Update CA_UsageDocument.md * Update CA-Tool--SBOM-Vision.md * Update README.md * IT cases refined * IT cases * IT failure * IT Changes * Updated README.md * IT failure resolved * IT Changes * Mavne Files for IT * Update appSettingsSW360IntegrationTest.json * IT Issues * IT test case fix * appsettings.json file update * Nuget Dev Initial changes * UT for Nuget Dev * Issue on UT * Dependency mapping for NPM (#59) * Update compile.yml * Update compile.yml * Update compile.yml * CI by vg_adapting angular16plusversion packagelockjson format in pkg identifier_15062023 * Update compile.yml * dependency mapping check in July7 * IT failure fix for dependencymapping * IT fix * Test case fix for IT * Unit test addtion * Integration test and Review chnages --------- Co-authored-by: Vijayalakshmi027 <58800320+Vijayalakshmi027@users.noreply.github.com> Co-authored-by: MadanReddyK <125262006+MadanReddyK@users.noreply.github.com> Co-authored-by: Viji * changes * IT Changes * PR Review comments added * case issue * nuspec file updated * Package Found issue * code change for devdependency mapping for maven * Nuget Changes * Nuspec update * SBOM/maven dev dependency change 17 july (#63) * Maven development Dependency Change * Unit test failure fix * Integration test case fix * Integration test file changes * Access specifier modification * Review comments * SBOM template reading initial * Dependencymappingfor maven unit and IT test cases (#64) * Nupec Nuget dll updates * error message update * Nuget Assets issue * CA.nuspec update * Unit test file updation * IT fix * Nuspec update * Nuspec update --------- Co-authored-by: Sumanth K B * SBOM UT added * UT cases added * IT added for SBOM Template * Sbom update * IT Updated * Usage Doc updated * Update * Update CA_UsageDocument.md * Dependency mappings * SBOM dependency mapping NuGet (#68) * Dependency mapping Nuget_Aug1 * Dependency Mapping for Nuget * Revert "Dependency Mapping for Nuget" This reverts commit a91fc7250b4f237d7a35ed4c974044f2cb3266bd. * Ut File change * Bug fix for Jfrog Repo identification * Dependency mapping Nuget_commitaug3 * Removed Unwanted methods * CycloneDx filepath read * Removed package lock parsing logic * appsettings update * File Rename * Review comments * Null check added for template * Review changes * Update * Python changes * Remove dev dependency flag addition while creating component (#75) * UT addition for remove devdependency flag * Remove dev dependency while creating component * code changes for maven sbom template adding properties. (#74) * code changes for maven sbom template adding properties. * maven method moved to commenhelper file --------- Co-authored-by: Chalapala RaghavendraReddy * Data Constant added * sonar issue fixed * sonarcube issues fixed * sonarcube issues fixed * Changes * Template update * Bug fixed for maven files parseing input components count * Fix: Integration Test case failure (#82) * Unit Test case addtion * IT failure fix * Method name change * UT failure fix * Sonar issue fixed (#79) Co-authored-by: Chalapala RaghavendraReddy * Updated Readme and UsageDoc/CA_UsageDocument.md (#78) * Updated Readme and UsageDoc/CA_UsageDocument.md * Nuspec update * Review comments update * Review comment updates * Changes for Jfrog Repo Identification * UT added * Code Smell Fixes (#84) * Unit Test case addtion * IT failure fix * Method name change * UT failure fix * Code Smells fix * IT failure fix * Method name change * Log updation * UT Updation * Coverage improvement * Coverage improvement * Code Smell fix * Coverage improvement * Code coverage improvement * Pythom processor test added * UT code change * Coverage improvement * Test case fix * Update PythonParserTests.cs * Code Coverage Added * Python Class Exluded from Coverage * UnChange CC * Python CodeCoverage * CC Change --------- Co-authored-by: Sumanth K B * Update CA_UsageDocument.md * Update pyproject.toml * Updated Readme OSS files * Bug fix for notimplemented exception * Bug Fix for component exclusion * Nuget Change * Docker file update --------- Co-authored-by: Sumanth K B Co-authored-by: Viji Co-authored-by: sumanthkb44 <84563853+sumanthkb44@users.noreply.github.com> Co-authored-by: Vijayalakshmi027 <58800320+Vijayalakshmi027@users.noreply.github.com> Co-authored-by: MadanReddyK <125262006+MadanReddyK@users.noreply.github.com> Co-authored-by: Chalapala RaghavendraReddy Co-authored-by: crvreddy <136427687+crvreddy@users.noreply.github.com> --- src/LCT.PackageIdentifier/MavenProcessor.cs | 4 ++ .../Model/NugetModel/BuildInfoComponent.cs | 2 +- .../NugetDevDependencyParser.cs | 37 +++++++++++++++++-- src/LCT.PackageIdentifier/NugetProcessor.cs | 11 +++++- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 78e31545..df303f0a 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -37,6 +37,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) List componentsToBOM = new(); List ListOfComponents = new(); Bom bom = new(); + int noOfExcludedComponents = 0; List dependenciesForBOM = new(); List configFiles; @@ -85,6 +86,9 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) componentsForBOM = ListOfComponents.Distinct(new ComponentEqualityComparer()).ToList(); BomCreator.bomKpiData.DuplicateComponents = totalComponentsIdentified - componentsForBOM.Count; + + componentsForBOM = CommonHelper.RemoveExcludedComponents(componentsForBOM, appSettings.Maven.ExcludedComponents, ref noOfExcludedComponents); + BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; bom.Components = componentsForBOM; bom.Dependencies = dependenciesForBOM; diff --git a/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs b/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs index 6c0cb5c8..19a801db 100644 --- a/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs +++ b/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs @@ -158,7 +158,7 @@ public override bool Equals(object obj) public override int GetHashCode() { - throw new NotImplementedException(); + return 0; } } } diff --git a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs index a113151a..55eaee8e 100644 --- a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs +++ b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs @@ -18,6 +18,7 @@ using System.Security.Cryptography; using LCT.PackageIdentifier.Model.NugetModel; using System.Text.Json; +using System.Runtime.InteropServices; namespace LCT.PackageIdentifier { @@ -122,9 +123,37 @@ private static void ParseJsonFile(string filePath, Container container) LockFile assetFile = assetFileReader.Read(filePath); if (assetFile.PackageSpec != null) { - isTestProject = IsTestProject(assetFile.PackageSpec.RestoreMetadata.ProjectPath); - - container.Name = Path.GetFileName(assetFile.PackageSpec.RestoreMetadata.ProjectPath); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Logger.Debug($"ParseJsonFile():Windows Asset FileName: " + assetFile.PackageSpec.RestoreMetadata.ProjectPath); + isTestProject = IsTestProject(assetFile.PackageSpec.RestoreMetadata.ProjectPath); + container.Name = Path.GetFileName(assetFile.PackageSpec.RestoreMetadata.ProjectPath); + Logger.Debug($"ParseJsonFile():Windows Asset File: IsTestProject: " + isTestProject); + } + else + { + string csprojFilePath = ""; + string dirName = Path.GetDirectoryName(filePath); + if (dirName.Contains("obj")) + { + dirName = dirName.Replace("obj", ""); + string[] filePaths = Directory.GetFiles(dirName, "*.csproj"); + csprojFilePath = filePaths.Length > 0 ? filePaths[0] : ""; + } + if(!string.IsNullOrEmpty(csprojFilePath) && File.Exists(csprojFilePath)) + { + Logger.Debug($"ParseJsonFile():Linux Asset FileName: " + csprojFilePath); + isTestProject = IsTestProject(csprojFilePath); + container.Name = Path.GetFileName(csprojFilePath); + Logger.Debug($"ParseJsonFile():Linux Asset File: IsTestProject: " + isTestProject); + } + else + { + Logger.Debug($"ParseJsonFile():Linux Asset FileName Not Found!! "); + isTestProject = false; + container.Name = Path.GetFileName(filePath); + } + } if (isTestProject) { @@ -138,6 +167,8 @@ private static void ParseJsonFile(string filePath, Container container) ParseLibrary(library, isTestProject, components, assetFile); } } + + Logger.Debug($"ParseJsonFile():Asset file found components: " + components.Count); } } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 9c70f2e8..d28cd959 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -392,7 +392,15 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List listofComponents = new(); ParseInputFiles(appSettings, filepath, listofComponents); ConvertToCycloneDXModel(listComponentForBOM, listofComponents, dependencies); - bom.Dependencies = dependencies; + if (bom.Dependencies == null) + { + bom.Dependencies = dependencies; + dependencies = new List(); + } + else + { + bom.Dependencies.AddRange(dependencies); + } BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = listComponentForBOM.Count; } } @@ -429,7 +437,6 @@ private static void GetDetailsforManuallyAdded(List componentsForBOM, private static void ConvertToCycloneDXModel(List listComponentForBOM, List listofComponents, List dependencies) { - foreach (var prop in listofComponents) { Component components = new Component From 0f29458a4f39d3923733cedb839e0eda67b45903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karthika=20Geethanand=20=20=C2=AF=5C=5F=28=E3=83=84=29=5F/?= =?UTF-8?q?=C2=AF?= <40568919+karthika-g@users.noreply.github.com> Date: Fri, 8 Sep 2023 19:16:40 +0530 Subject: [PATCH 03/18] Update compile.yml --- .github/workflows/compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index ed87c95c..c46e47ac 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -116,7 +116,7 @@ jobs: DOCKERDEVARTIFACTORY: ${{ secrets.DOCKERDEVARTIFACTORY }} - name: Test - #if: ${{ false }} # disable for now + if: ${{ false }} # disable for now run: | $TestProjects = Get-ChildItem -Path *test*.csproj -Recurse -exclude TestUtilities.csproj,UnitTestUtilities.csproj Write-Host "**************************The test projects considered for execution: $TestProjects ******************************" From ebdcfae9fd08b2f078c2eece2c439e297d63eec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karthika=20Geethanand=20=20=C2=AF=5C=5F=28=E3=83=84=29=5F/?= =?UTF-8?q?=C2=AF?= <40568919+karthika-g@users.noreply.github.com> Date: Fri, 8 Sep 2023 19:59:26 +0530 Subject: [PATCH 04/18] Update CA.nuspec --- CA.nuspec | 1 + 1 file changed, 1 insertion(+) diff --git a/CA.nuspec b/CA.nuspec index c6c712e3..68126752 100644 --- a/CA.nuspec +++ b/CA.nuspec @@ -82,6 +82,7 @@ + From ecc6bbf0c7bb2569a87dc3e5e7ea3b06ff28c1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karthika=20Geethanand=20=20=C2=AF=5C=5F=28=E3=83=84=29=5F/?= =?UTF-8?q?=C2=AF?= <40568919+karthika-g@users.noreply.github.com> Date: Fri, 8 Sep 2023 20:08:44 +0530 Subject: [PATCH 05/18] Update MavenProcessor.cs --- src/LCT.PackageIdentifier/MavenProcessor.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LCT.PackageIdentifier/MavenProcessor.cs b/src/LCT.PackageIdentifier/MavenProcessor.cs index 8c57a6f6..2ba4d133 100644 --- a/src/LCT.PackageIdentifier/MavenProcessor.cs +++ b/src/LCT.PackageIdentifier/MavenProcessor.cs @@ -87,8 +87,6 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) BomCreator.bomKpiData.DuplicateComponents = totalComponentsIdentified - componentsForBOM.Count; - componentsForBOM = CommonHelper.RemoveExcludedComponents(componentsForBOM, appSettings.Maven.ExcludedComponents, ref noOfExcludedComponents); - BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; if (appSettings.Maven.ExcludedComponents != null) { From adb8575043e2b4741f442a42ddf0b8d89dbb05d3 Mon Sep 17 00:00:00 2001 From: MadanReddyK <125262006+MadanReddyK@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:00:03 +0530 Subject: [PATCH 06/18] Update compile.yml --- .github/workflows/compile.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index c46e47ac..0ef64c47 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -3,8 +3,7 @@ name: Build & Test on: push: pull_request: - schedule: - - cron: "30 0 * * 5" + jobs: From daaf2c048b6aebd61ac7cd34cfb19a12f902439e Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Tue, 12 Sep 2023 11:19:42 +0530 Subject: [PATCH 07/18] Creator changes --- .../Model/NugetModel/BuildInfoComponent.cs | 2 +- .../CreatorHelperTest.cs | 79 ++++++++++++++----- .../PythonPackageCreatorTest.cs | 66 ++++++++++++++++ .../UrlHelperTest.cs | 26 ++++++ .../Interfaces/IURLHelper.cs | 8 ++ src/LCT.SW360PackageCreator/URLHelper.cs | 9 ++- src/LCT.Services/Sw360CommonService.cs | 4 + 7 files changed, 174 insertions(+), 20 deletions(-) create mode 100644 src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs diff --git a/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs b/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs index 6c0cb5c8..19a801db 100644 --- a/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs +++ b/src/LCT.PackageIdentifier/Model/NugetModel/BuildInfoComponent.cs @@ -158,7 +158,7 @@ public override bool Equals(object obj) public override int GetHashCode() { - throw new NotImplementedException(); + return 0; } } } diff --git a/src/LCT.SW360PackageCreator.UTest/CreatorHelperTest.cs b/src/LCT.SW360PackageCreator.UTest/CreatorHelperTest.cs index 9de3e80d..9a88cf03 100644 --- a/src/LCT.SW360PackageCreator.UTest/CreatorHelperTest.cs +++ b/src/LCT.SW360PackageCreator.UTest/CreatorHelperTest.cs @@ -18,6 +18,7 @@ using CycloneDX.Models; using System.Diagnostics; using LCT.Common; +using Castle.Core.Internal; namespace SW360ComponentCreator.UTest { @@ -47,9 +48,11 @@ public void Test_GetDownloadUrlNotFoundList() } }; - IDictionary _packageDownloderList = new Dictionary(); - _packageDownloderList.Add("NPM", new PackageDownloader()); - _packageDownloderList.Add("NUGET", new PackageDownloader()); + IDictionary _packageDownloderList = new Dictionary + { + { "NPM", new PackageDownloader() }, + { "NUGET", new PackageDownloader() } + }; var creatorHelper = new CreatorHelper(_packageDownloderList); var actual = creatorHelper.GetDownloadUrlNotFoundList(lstComparisonBomData); @@ -68,15 +71,16 @@ public async Task DownloadReleaseAttachmentSource_ForNPMPackage_ReturnSuccess() SourceUrl = "https://github.com/angular/angular.git", DownloadUrl = "https://github.com/angular/angular.git" }; - IDictionary _packageDownloderList = new Dictionary(); - _packageDownloderList.Add("NPM", new PackageDownloader()); - _packageDownloderList.Add("NUGET", new PackageDownloader()); + IDictionary _packageDownloderList = new Dictionary + { + { "NPM", new PackageDownloader() }, + { "NUGET", new PackageDownloader() } + }; var creatorHelper = new CreatorHelper(_packageDownloderList); //Act var attachmentUrlList = await creatorHelper.DownloadReleaseAttachmentSource(lstComparisonBomData); - //Assert Assert.That(attachmentUrlList.ContainsKey("SOURCE")); } @@ -97,13 +101,8 @@ public void InitializeSw360ProjectService_ForGivenAppSeetings_ReturnsSw360Servic appSettings.SW360AuthTokenType = "Token"; appSettings.Sw360Token = "uifhiopsjfposddkf[fopefp[ld[p[lfffuhdffdkf"; appSettings.SW360URL = "http://localhost:8090"; - //IDictionary _packageDownloderList = new Dictionary(); - //_packageDownloderList.Add("NPM", new PackageDownloader()); - //_packageDownloderList.Add("NUGET", new PackageDownloader()); - //var creatorHelper = new CreatorHelper(_packageDownloderList); // Act - ISw360ProjectService sw360ProjectService = CreatorHelper.InitializeSw360ProjectService(appSettings); // assert @@ -118,10 +117,6 @@ public void InitializeSw360CreatorService_ForGivenAppSettings_ReturnsSw360Creato appSettings.SW360AuthTokenType = "Token"; appSettings.Sw360Token = "uifhiopsjfposddkf[fopefp[ld[p[lfffuhdffdkf"; appSettings.SW360URL = "http://localhost:8090"; - //IDictionary _packageDownloderList = new Dictionary(); - //_packageDownloderList.Add("NPM", new PackageDownloader()); - //_packageDownloderList.Add("NUGET", new PackageDownloader()); - //var creatorHelper = new CreatorHelper(_packageDownloderList); // Act ISw360CreatorService sw360CreatorService = CreatorHelper.InitializeSw360CreatorService(appSettings); @@ -130,8 +125,6 @@ public void InitializeSw360CreatorService_ForGivenAppSettings_ReturnsSw360Creato Assert.That(sw360CreatorService, Is.Not.Null); } - - [Test] public async Task SetContentsForComparisonBOM_ProvidedValidBomDetails_ReturnsListOfComparisonBomData() { @@ -309,5 +302,55 @@ public void WriteSourceNotFoundListToConsole_GetComparisionBomDatan_ReturnsNothi Assert.IsTrue(true); } + [Test] + public async Task DownloadReleaseAttachmentSource_ForPythonPackage_ReturnSuccess() + { + //Arrange + var lstComparisonBomData = new ComparisonBomData() + { + Name = "cachecontrol", + Version = "0.12.11", + ReleaseExternalId = "pkg:pypi/cachecontrol@0.12.11", + SourceUrl = "https://files.pythonhosted.org/packages/46/9b/34215200b0c2b2229d7be45c1436ca0e8cad3b10de42cfea96983bd70248/CacheControl-0.12.11.tar.gz", + DownloadUrl = "https://files.pythonhosted.org/packages/46/9b/34215200b0c2b2229d7be45c1436ca0e8cad3b10de42cfea96983bd70248/CacheControl-0.12.11.tar.gz" + }; + IDictionary _packageDownloderList = new Dictionary + { + { "PYTHON", new PackageDownloader() } + }; + var creatorHelper = new CreatorHelper(_packageDownloderList); + + //Act + var attachmentUrlList = await creatorHelper.DownloadReleaseAttachmentSource(lstComparisonBomData); + + //Assert + Assert.That(attachmentUrlList.ContainsKey("SOURCE")); + } + + [Test] + public async Task DownloadReleaseAttachmentSource_ForPythonPackage_ReturnFailure() + { + //Arrange + var lstComparisonBomData = new ComparisonBomData() + { + Name = "cachecontrol22", + Version = "0.12.1122", + ReleaseExternalId = "pkg:pypi/cachecontrol22@0.12.1122", + SourceUrl = "https://files.pythonhosted.org/packages/46/9b/34215200b0c2b2229d7be45c1436ca0e8cad3b10de42cfea96983bd70248/CacheControl22-0.12.1122.tar.gz", + DownloadUrl = "https://files.pythonhosted.org/packages/46/9b/34215200b0c2b2229d7be45c1436ca0e8cad3b10de42cfea96983bd70248/CacheControl22-0.12.1122.tar.gz" + }; + IDictionary _packageDownloderList = new Dictionary + { + { "PYTHON", new PackageDownloader() } + }; + var creatorHelper = new CreatorHelper(_packageDownloderList); + + //Act + var attachmentUrlList = await creatorHelper.DownloadReleaseAttachmentSource(lstComparisonBomData); + + //Assert + Assert.That(attachmentUrlList.IsNullOrEmpty); + } + } } diff --git a/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs b/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs new file mode 100644 index 00000000..374a3966 --- /dev/null +++ b/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs @@ -0,0 +1,66 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using LCT.Common.Model; +using LCT.SW360PackageCreator.Interfaces; +using Moq; +using NUnit.Framework; +using System.IO; +using System.Threading.Tasks; + +namespace LCT.SW360PackageCreator.UTest +{ + [TestFixture] + class PythonPackageCreatorTest + { + [TestCase] + public async Task DownloadReleaseAttachmentSourceForDebian_ProvidedSourceURL_ReturnsDownloadPath() + { + //Arrange + var lstComparisonBomData = new ComparisonBomData() + { + Name = "adduser", + Version = "3.118", + DownloadUrl = "", + SourceUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xz", + ReleaseExternalId = "pkg:deb/debian/adduser@3.1182?arch=source" + }; + var localPathforDownload = $"{Path.GetTempPath()}/ClearingTool/DownloadedFiles/"; + var debianPatcher = new Mock(); + var debianPackageDownloader = new DebianPackageDownloader(debianPatcher.Object); + + //Act + var attachmentUrlList = await debianPackageDownloader.DownloadPackage(lstComparisonBomData, localPathforDownload); + + //Assert + Assert.IsNotEmpty(attachmentUrlList); + } + + [TestCase] + public async Task DownloadReleaseAttachmentSourceForDebian_ProvidedInValidSourceURL_ReturnsNull() + { + //Arrange + //Sending Incorrect SourceURL + var lstComparisonBomData = new ComparisonBomData() + { + Name = "adduser", + Version = "3.118", + DownloadUrl = "", + SourceUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xx", + ReleaseExternalId = "pkg:deb/debian/adduser@3.1182?arch=source" + }; + var localPathforDownload = $"{Path.GetTempPath()}/ClearingTool/DownloadedFiles/"; + var debianPatcher = new Mock(); + var debianPackageDownloader = new DebianPackageDownloader(debianPatcher.Object); + + //Act + var attachmentUrlList = await debianPackageDownloader.DownloadPackage(lstComparisonBomData, localPathforDownload); + + //Assert + Assert.IsEmpty(attachmentUrlList); + } + } +} diff --git a/src/LCT.SW360PackageCreator.UTest/UrlHelperTest.cs b/src/LCT.SW360PackageCreator.UTest/UrlHelperTest.cs index 0fbc04aa..1b0e6ab0 100644 --- a/src/LCT.SW360PackageCreator.UTest/UrlHelperTest.cs +++ b/src/LCT.SW360PackageCreator.UTest/UrlHelperTest.cs @@ -104,5 +104,31 @@ public void GetReleaseExternalId_ProvidedPackagDetails_ReturnsEncodedExternalID( Assert.That(sourceUrlDetails, Is.EqualTo("pkg:deb/debian/gnutls28@3.6.7-4%2Bdeb10u7?arch=source")); } + + [TestCase("cachecontrol", "0.12.11")] + public async Task GetSourceUrlForPythonPackage_ProvidedPackageDetails_ReturnsValidSourceURL(string componentName, string version) + { + // Arrange + IUrlHelper urlHelper = new UrlHelper(); + + // Act + string sourceUrl = await urlHelper.GetSourceUrlForPythonPackage(componentName, version); + + // Assert + Assert.That(sourceUrl, Is.EqualTo("https://files.pythonhosted.org/packages/46/9b/34215200b0c2b2229d7be45c1436ca0e8cad3b10de42cfea96983bd70248/CacheControl-0.12.11.tar.gz")); + } + + [TestCase("cachecontrol22", "0.12.111")] + public async Task GetSourceUrlForPythonPackage_ProvidedInvalidPackageDetails_ReturnsEmptyString(string componentName, string version) + { + // Arrange + IUrlHelper urlHelper = new UrlHelper(); + + // Act + string sourceUrl = await urlHelper.GetSourceUrlForPythonPackage(componentName, version); + + // Assert + Assert.That(string.IsNullOrEmpty(sourceUrl)); + } } } diff --git a/src/LCT.SW360PackageCreator/Interfaces/IURLHelper.cs b/src/LCT.SW360PackageCreator/Interfaces/IURLHelper.cs index 636ccae6..33fe37d7 100644 --- a/src/LCT.SW360PackageCreator/Interfaces/IURLHelper.cs +++ b/src/LCT.SW360PackageCreator/Interfaces/IURLHelper.cs @@ -39,5 +39,13 @@ public interface IUrlHelper /// /// string string GetSourceUrlForNpmPackage(string componentName, string version); + + /// + /// Gets the Source URL for PYTHON Package + /// + /// + /// + /// string + Task GetSourceUrlForPythonPackage(string componentName, string componenVersion); } } diff --git a/src/LCT.SW360PackageCreator/URLHelper.cs b/src/LCT.SW360PackageCreator/URLHelper.cs index 89dbc62d..26141a4f 100644 --- a/src/LCT.SW360PackageCreator/URLHelper.cs +++ b/src/LCT.SW360PackageCreator/URLHelper.cs @@ -440,6 +440,13 @@ private async Task RetryToGetSourceURlDetailsAsync(string name, s return sourceDetails; } + /// + /// Gets the Source Url For Python Package + /// + /// + /// + /// + /// string public async Task GetSourceUrlForPythonPackage(string componentName, string componenVersion) { Logger.Debug($"URLHelper.GetSourceUrlForPythonPackage():Start"); @@ -550,7 +557,6 @@ public static string GetCorrectFileExtension(string sourceURL) return fullname; } - public void Dispose() { Dispose(true); @@ -571,5 +577,6 @@ protected virtual void Dispose(bool disposing) _disposed = true; } + } } diff --git a/src/LCT.Services/Sw360CommonService.cs b/src/LCT.Services/Sw360CommonService.cs index c3187855..67875598 100644 --- a/src/LCT.Services/Sw360CommonService.cs +++ b/src/LCT.Services/Sw360CommonService.cs @@ -148,6 +148,10 @@ public async Task GetReleaseDataByExternalId(string releaseName, { Logger.Error($"GetReleaseDataByExternalId():", ex); } + catch (JsonReaderException ex) + { + Logger.Error($"GetReleaseDataByExternalId():JsonReaderException", ex); + } return releasestatus; } From 29b7157dd744e8103d2c9c65040a12dc8c13c097 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Wed, 13 Sep 2023 15:36:51 +0530 Subject: [PATCH 08/18] UT & IT Added --- .../Python/PythonSample.cdx.json | 303 ++++++++++++++++++ .../PackageUploadHelperTest.cs | 18 +- .../PackageUploaderTest.cs | 2 - src/LCT.PackageIdentifier/NugetProcessor.cs | 9 +- .../PythonPackageCreatorTest.cs | 66 ---- .../Maven/ArtifactoryUploaderMaven.cs | 1 - .../NPM/ArtifactoryUploaderNpm.cs | 11 - .../Python/CCTComparisonBOMPythonInitial.json | 252 +++++++++++++++ .../Python/CCTLocalBOMPythonInitial.json | 204 ++++++++++++ .../Python/ComponentCreatorInitialPython.cs | 168 ++++++++++ .../Python/PackageIdentifierInitialPython.cs | 95 ++++++ .../SW360IntegrationTest.csproj | 6 + 12 files changed, 1035 insertions(+), 100 deletions(-) create mode 100644 TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Python/PythonSample.cdx.json delete mode 100644 src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs create mode 100644 src/SW360IntegrationTest/PackageCreatorTestFiles/Python/CCTComparisonBOMPythonInitial.json create mode 100644 src/SW360IntegrationTest/PackageIdentifierTestFiles/Python/CCTLocalBOMPythonInitial.json create mode 100644 src/SW360IntegrationTest/Python/ComponentCreatorInitialPython.cs create mode 100644 src/SW360IntegrationTest/Python/PackageIdentifierInitialPython.cs diff --git a/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Python/PythonSample.cdx.json b/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Python/PythonSample.cdx.json new file mode 100644 index 00000000..5b3992cd --- /dev/null +++ b/TestFiles/IntegrationTestFiles/SystemTest1stIterationData/Python/PythonSample.cdx.json @@ -0,0 +1,303 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:f00c34f8-f231-4840-a23d-d3beb1ef8a49", + "version": 1, + "metadata": { + "timestamp": "2023-04-20T04:39:27Z", + "tools": [ + { + "vendor": "anchore", + "name": "syft", + "version": "0.76.0" + } + ], + "component": { + "bom-ref": "f78e7117441425e1", + "type": "file", + "name": "/mnt/InputImages/requirements.txt" + } + }, + "components": [ + { + "bom-ref": "pkg:pypi/html5lib@6.0?package-id=f6e2935343960313", + "type": "library", + "name": "html5lib", + "version": "1.1", + "cpe": "cpe:2.3:a:python-PyYAML:python-PyYAML:6.0:*:*:*:*:*:*:*", + "purl": "pkg:pypi/html5lib@6.0", + "properties": [ + { + "name": "syft:package:foundBy", + "value": "python-index-cataloger" + }, + { + "name": "syft:package:language", + "value": "python" + }, + { + "name": "syft:package:type", + "value": "python" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-PyYAML:python_PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_PyYAML:python-PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_PyYAML:python_PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:PyYAML:python-PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:PyYAML:python_PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-PyYAML:PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python-PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python_PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_PyYAML:PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:PyYAML:PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:PyYAML:6.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:location:0:path", + "value": "/mnt/InputImages/requirements.txt" + } + ] + }, + { + "bom-ref": "pkg:pypi/attrs@22.1.0?package-id=96867fd45240c838", + "type": "library", + "name": "attrs", + "version": "22.1.0", + "cpe": "cpe:2.3:a:python-attrs:python-attrs:22.1.0:*:*:*:*:*:*:*", + "purl": "pkg:pypi/attrs@22.1.0", + "properties": [ + { + "name": "syft:package:foundBy", + "value": "python-index-cataloger" + }, + { + "name": "syft:package:language", + "value": "python" + }, + { + "name": "syft:package:type", + "value": "python" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-attrs:python_attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_attrs:python-attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_attrs:python_attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python-attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python_attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:attrs:python-attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:attrs:python_attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-attrs:attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_attrs:attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:attrs:attrs:22.1.0:*:*:*:*:*:*:*" + }, + { + "name": "syft:location:0:path", + "value": "/mnt/InputImages/requirements.txt" + } + ] + }, + { + "bom-ref": "pkg:pypi/beautifulsoup4@4.11.1?package-id=ae34b56b6d249689", + "type": "library", + "name": "beautifulsoup4", + "version": "4.11.1", + "cpe": "cpe:2.3:a:python-beautifulsoup4:python-beautifulsoup4:4.11.1:*:*:*:*:*:*:*", + "purl": "pkg:pypi/beautifulsoup4@4.11.1", + "properties": [ + { + "name": "syft:package:foundBy", + "value": "python-index-cataloger" + }, + { + "name": "syft:package:language", + "value": "python" + }, + { + "name": "syft:package:type", + "value": "python" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-beautifulsoup4:python_beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_beautifulsoup4:python-beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_beautifulsoup4:python_beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:beautifulsoup4:python-beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:beautifulsoup4:python_beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-beautifulsoup4:beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_beautifulsoup4:beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:beautifulsoup4:beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python-beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python_beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:beautifulsoup4:4.11.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:location:0:path", + "value": "/mnt/InputImages/requirements.txt" + } + ] + }, + { + "bom-ref": "pkg:pypi/certifi@2022.6.15?package-id=58b26abc57776bea", + "type": "library", + "name": "certifi", + "version": "2022.6.15", + "cpe": "cpe:2.3:a:python-certifi:python-certifi:2022.6.15:*:*:*:*:*:*:*", + "purl": "pkg:pypi/certifi@2022.6.15", + "properties": [ + { + "name": "syft:package:foundBy", + "value": "python-index-cataloger" + }, + { + "name": "syft:package:language", + "value": "python" + }, + { + "name": "syft:package:type", + "value": "python" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-certifi:python_certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_certifi:python-certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_certifi:python_certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:certifi:python-certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:certifi:python_certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-certifi:certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_certifi:certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python-certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python_certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:certifi:certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:certifi:2022.6.15:*:*:*:*:*:*:*" + }, + { + "name": "syft:location:0:path", + "value": "/mnt/InputImages/requirements.txt" + } + ] + } + ] +} diff --git a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs index 107abc2c..f21526b4 100644 --- a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs +++ b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs @@ -32,17 +32,15 @@ public void GetComponentListFromComparisonBOM_GivenComparisonBOM_ReturnsComponen // Assert Assert.That(11, Is.EqualTo(componentList.Components.Count), "Checks for no of components"); } + [Test] public void GetComponentListFromComparisonBOM_GivenInvalidComparisonBOM_ReturnsException() { //Arrange string comparisonBOMPath = @"TestFiles\CCTComparisonBOM.json"; - //Act && Assert Assert.Throws(() => PackageUploadHelper.GetComponentListFromComparisonBOM(comparisonBOMPath)); - - } [Test] public void GetComponentListFromComparisonBOM_GivenInvalidfile_ReturnsException() @@ -54,8 +52,6 @@ public void GetComponentListFromComparisonBOM_GivenInvalidfile_ReturnsException( //Act && Assert Assert.Throws(() => PackageUploadHelper.GetComponentListFromComparisonBOM(comparisonBOMPath)); - - } @@ -88,7 +84,6 @@ public void GetComponentsToBeUploadedToArtifactory_GivenAllApprovedComponentList { //Arrange List componentLists = GetComponentList(); - foreach (var component in componentLists) { if (component.Name == "@angular/core") @@ -109,8 +104,10 @@ public void GetComponentsToBeUploadedToArtifactory_GivenAllApprovedComponentList LogFolderPath= outFolder }; string LogfolderPath = appSettings.LogFolderPath; + //Act List uploadList = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); + // Assert Assert.That(4, Is.EqualTo(uploadList.Count), "Checks for 3 no of components to upload"); } @@ -121,10 +118,7 @@ public void GetComponentsToBeUploadedToArtifactory_GivenNotApprovedComponentList List componentLists = GetComponentList(); foreach (var component in componentLists) { - component.Properties[1].Value = "NEW_CLEARING"; - - } CommonAppSettings appSettings = new CommonAppSettings() @@ -138,6 +132,7 @@ public void GetComponentsToBeUploadedToArtifactory_GivenNotApprovedComponentList //Act List uploadList = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); + // Assert Assert.That(0, Is.EqualTo(uploadList.Count), "Checks for components to upload to be zero"); } @@ -212,12 +207,7 @@ private static List GetComponentList() comp4.Properties.Add(propinternal); comp4.Properties.Add(prop3); componentLists.Add(comp4); - - return componentLists; } - - - } } diff --git a/src/AritfactoryUploader.UTest/PackageUploaderTest.cs b/src/AritfactoryUploader.UTest/PackageUploaderTest.cs index 5b49f44e..3b53fbbc 100644 --- a/src/AritfactoryUploader.UTest/PackageUploaderTest.cs +++ b/src/AritfactoryUploader.UTest/PackageUploaderTest.cs @@ -45,8 +45,6 @@ public async Task UploadPackageToArtifactory_GivenAppsettings() await PackageUploader.UploadPackageToArtifactory(CommonAppSettings); // Assert - - Assert.That(11, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesToBeUploaded), "Checks for no of components"); } } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index d28cd959..0368462f 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -86,7 +86,6 @@ public static List ParsePackageConfig(string packagesFilePath, Com if (IsDevDependent(referenceList, name, version) || devDependencyAttribute?.Value != null) { - BomCreator.bomKpiData.DevDependentComponents++; isDev = "true"; } @@ -129,7 +128,6 @@ public static bool IsDevDependent(List referenceDetails, strin { if (item.Library == name && item.Version == version && item.Private) { - return true; } } @@ -519,14 +517,13 @@ private static void ParseInputFiles(CommonAppSettings appSettings, string filepa } else if (filepath.EndsWith(".config")) { - listofComponents.AddRange(ParsePackageConfig(filepath, appSettings)); + var list = ParsePackageConfig(filepath, appSettings); + listofComponents.AddRange(list); } else { - var list = ParsePackageConfig(filepath, appSettings); - listofComponents.AddRange(list); + Logger.Warn("No Proper input files found for Nuget package types."); } - } diff --git a/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs b/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs deleted file mode 100644 index 374a3966..00000000 --- a/src/LCT.SW360PackageCreator.UTest/PythonPackageCreatorTest.cs +++ /dev/null @@ -1,66 +0,0 @@ -// -------------------------------------------------------------------------------------------------------------------- -// SPDX-FileCopyrightText: 2023 Siemens AG -// -// SPDX-License-Identifier: MIT -// -------------------------------------------------------------------------------------------------------------------- - -using LCT.Common.Model; -using LCT.SW360PackageCreator.Interfaces; -using Moq; -using NUnit.Framework; -using System.IO; -using System.Threading.Tasks; - -namespace LCT.SW360PackageCreator.UTest -{ - [TestFixture] - class PythonPackageCreatorTest - { - [TestCase] - public async Task DownloadReleaseAttachmentSourceForDebian_ProvidedSourceURL_ReturnsDownloadPath() - { - //Arrange - var lstComparisonBomData = new ComparisonBomData() - { - Name = "adduser", - Version = "3.118", - DownloadUrl = "", - SourceUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xz", - ReleaseExternalId = "pkg:deb/debian/adduser@3.1182?arch=source" - }; - var localPathforDownload = $"{Path.GetTempPath()}/ClearingTool/DownloadedFiles/"; - var debianPatcher = new Mock(); - var debianPackageDownloader = new DebianPackageDownloader(debianPatcher.Object); - - //Act - var attachmentUrlList = await debianPackageDownloader.DownloadPackage(lstComparisonBomData, localPathforDownload); - - //Assert - Assert.IsNotEmpty(attachmentUrlList); - } - - [TestCase] - public async Task DownloadReleaseAttachmentSourceForDebian_ProvidedInValidSourceURL_ReturnsNull() - { - //Arrange - //Sending Incorrect SourceURL - var lstComparisonBomData = new ComparisonBomData() - { - Name = "adduser", - Version = "3.118", - DownloadUrl = "", - SourceUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xx", - ReleaseExternalId = "pkg:deb/debian/adduser@3.1182?arch=source" - }; - var localPathforDownload = $"{Path.GetTempPath()}/ClearingTool/DownloadedFiles/"; - var debianPatcher = new Mock(); - var debianPackageDownloader = new DebianPackageDownloader(debianPatcher.Object); - - //Act - var attachmentUrlList = await debianPackageDownloader.DownloadPackage(lstComparisonBomData, localPathforDownload); - - //Assert - Assert.IsEmpty(attachmentUrlList); - } - } -} diff --git a/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs b/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs index f32553db..8b4a4e13 100644 --- a/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs +++ b/src/SW360IntegrationTest/Maven/ArtifactoryUploaderMaven.cs @@ -6,7 +6,6 @@ // -------------------------------------------------------------------------------------------------------------------- using CycloneDX.Models; using NUnit.Framework; -using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; diff --git a/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs b/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs index a65d18d8..9bfafeba 100644 --- a/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs +++ b/src/SW360IntegrationTest/NPM/ArtifactoryUploaderNpm.cs @@ -42,8 +42,6 @@ public void ComponentUpload_IsUnsuccessful_AlreadyPresentInDestination() string comparisonBOMPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\ArtifactoryUploaderTestData\NPMComparisonBOM.json"; if (File.Exists(comparisonBOMPath)) { - - ComponentJsonParsor expected = new ComponentJsonParsor(); expected.Read(comparisonBOMPath); @@ -52,13 +50,9 @@ public void ComponentUpload_IsUnsuccessful_AlreadyPresentInDestination() Component components = item; if (components.Properties[3].Name.Contains("ApprovedStatus")) { - // Assert Assert.AreEqual("siparty-release-npm-egll", components.Properties[1].Value); - - } - } } } @@ -69,20 +63,15 @@ public void ComponentUpload_IsUnsuccessful_AlreadyPresentInDestination() public void ComponentUpload_IsFailure() { HttpClient httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.Add(TestConstant.JFrog_API_Header, testParameters.ArtifactoryUploadApiKey); httpClient.DefaultRequestHeaders.Add(TestConstant.Email, testParameters.ArtifactoryUploadUser); // Act string url = $"{TestConstant.JfrogApi}/@angular/core/-/core-9.1.3.tgz"; - HttpResponseMessage responseBody = httpClient.GetAsync(url).Result; - // Assert Assert.That(HttpStatusCode.NotFound, Is.EqualTo(responseBody.StatusCode), "Returns Failure status code"); - - } } diff --git a/src/SW360IntegrationTest/PackageCreatorTestFiles/Python/CCTComparisonBOMPythonInitial.json b/src/SW360IntegrationTest/PackageCreatorTestFiles/Python/CCTComparisonBOMPythonInitial.json new file mode 100644 index 00000000..50ca69e4 --- /dev/null +++ b/src/SW360IntegrationTest/PackageCreatorTestFiles/Python/CCTComparisonBOMPythonInitial.json @@ -0,0 +1,252 @@ +{ + "BomFormat": "CycloneDX", + "SpecVersion": 4, + "SpecVersionString": "1.4", + "SerialNumber": null, + "Version": null, + "Metadata": { + "Tools": [ + { + "Vendor": "Siemens AG", + "Name": "Clearing Automation Tool", + "Version": "4.0.0", + "Hashes": null + } + ], + "Authors": null, + "Component": null, + "Manufacture": null, + "Supplier": null + }, + "Components": [ + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/html5lib@1.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "html5lib", + "Version": "1.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/html5lib@1.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Not Available" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/131c06a0c522488ebc8fbe9162529094" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/attrs@22.1.0", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "attrs", + "Version": "22.1.0", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/attrs@22.1.0", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Not Available" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/c58e917444084bfa8955485c151cf945" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/beautifulsoup4@4.11.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "beautifulsoup4", + "Version": "4.11.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/beautifulsoup4@4.11.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Not Available" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/35dd2b7677774c139cea87df7af3fe8e" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/certifi@2022.6.15", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "certifi", + "Version": "2022.6.15", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/certifi@2022.6.15", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Not Available" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://localhost:8090/resource/api/releases/46fe1664a0b7424fa9c8db018f66cd50" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + } + ], + "Compositions": null +} \ No newline at end of file diff --git a/src/SW360IntegrationTest/PackageIdentifierTestFiles/Python/CCTLocalBOMPythonInitial.json b/src/SW360IntegrationTest/PackageIdentifierTestFiles/Python/CCTLocalBOMPythonInitial.json new file mode 100644 index 00000000..16dad124 --- /dev/null +++ b/src/SW360IntegrationTest/PackageIdentifierTestFiles/Python/CCTLocalBOMPythonInitial.json @@ -0,0 +1,204 @@ +{ + "BomFormat": "CycloneDX", + "SpecVersion": 4, + "SpecVersionString": "1.4", + "SerialNumber": null, + "Version": null, + "Metadata": { + "Tools": [ + { + "Vendor": "Siemens AG", + "Name": "Clearing Automation Tool", + "Version": "4.0.0", + "Hashes": null + } + ], + "Authors": null, + "Component": null, + "Manufacture": null, + "Supplier": null + }, + "Components": [ + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/html5lib@1.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "html5lib", + "Version": "1.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/html5lib@1.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/attrs@22.1.0", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "attrs", + "Version": "22.1.0", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/attrs@22.1.0", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/beautifulsoup4@4.11.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "beautifulsoup4", + "Version": "4.11.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/beautifulsoup4@4.11.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/certifi@2022.6.15", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "certifi", + "Version": "2022.6.15", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/certifi@2022.6.15", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + } + ], + "Evidence": null + } + ], + "Compositions": null +} \ No newline at end of file diff --git a/src/SW360IntegrationTest/Python/ComponentCreatorInitialPython.cs b/src/SW360IntegrationTest/Python/ComponentCreatorInitialPython.cs new file mode 100644 index 00000000..86938d3d --- /dev/null +++ b/src/SW360IntegrationTest/Python/ComponentCreatorInitialPython.cs @@ -0,0 +1,168 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (C) Siemens AG 2023. All rights reserved. +// +//MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using LCT.APICommunications.Model; +using Newtonsoft.Json; +using NUnit.Framework; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using TestUtilities; + +namespace SW360IntegrationTest.Python +{ + [TestFixture, Order(24)] + public class ComponentCreatorInitialPython + { + private TestParamNuget testParameters; + + [SetUp] + public void Setup() + { + testParameters = new TestParamNuget(); + OutFolder = TestHelper.OutFolder; + CCTComparisonBomTestFile = OutFolder + @"..\..\..\src\SW360IntegrationTest\PackageCreatorTestFiles\Python\CCTComparisonBOMPythonInitial.json"; + + if (!TestHelper.BOMCreated) + { + OutFolder = TestHelper.OutFolder; + string packagejsonPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\SystemTest1stIterationData\Python"; + string bomPath = OutFolder + @"\..\BOMs"; + TestHelper.RunBOMCreatorExe(new string[]{ + TestConstant.PackageFilePath, packagejsonPath, + TestConstant.BomFolderPath, bomPath, + TestConstant.Sw360Token, testParameters.SW360AuthTokenValue, + TestConstant.SW360AuthTokenType, testParameters.SW360AuthTokenType, + TestConstant.SW360URL, testParameters.SW360URL, + TestConstant.SW360ProjectID, testParameters.SW360ProjectID, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, + TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, + TestConstant.ProjectType,"PYTHON", + TestConstant.Mode,""}); + } + } + + [Test, Order(1)] + public void ComponentCreatorExe_ProvidedBOMFilePath_ReturnsSuccess() + { + string bomPath = OutFolder + $"\\..\\BOMs\\{testParameters.SW360ProjectName}_Bom.cdx.json"; + // Assert + // Check exit is normal + int returnValue = TestHelper.RunComponentCreatorExe(new string[] { + TestConstant.BomFilePath,bomPath, + TestConstant.Sw360Token, testParameters.SW360AuthTokenValue, + TestConstant.SW360URL, testParameters.SW360URL, + TestConstant.SW360AuthTokenType, testParameters.SW360AuthTokenType, + TestConstant.SW360ProjectID, testParameters.SW360ProjectID, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, + TestConstant.Mode,""}); + + Assert.IsTrue(returnValue == 0 || returnValue == 2, "Test to run Package Creator EXE execution"); + } + + [Test, Order(2)] + public void TestComparisionBOMUpdation() + { + // Expected + bool filecheck = false; + ComponentJsonParsor expected = new ComponentJsonParsor(); + expected.Read(CCTComparisonBomTestFile); + + // Actual + string generatedBOM = OutFolder + $"\\..\\BOMs\\{testParameters.SW360ProjectName}_Bom.cdx.json"; + if (File.Exists(generatedBOM)) + { + filecheck = true; + ComponentJsonParsor actual = new ComponentJsonParsor(); + actual.Read(generatedBOM); + + foreach (var item in expected.Components) + { + foreach (var i in actual.Components) + { + if ((i.Name == item.Name) && (i.Version == item.Version)) + { + Component component = i; + Assert.AreEqual(item.Name, component.Name); + Assert.AreEqual(item.Version, component.Version); + Assert.AreEqual(item.BomRef, component.BomRef); + Assert.AreEqual(item.Purl, component.Purl); + Assert.AreEqual(item.Properties.Count, component.Properties.Count); + } + } + } + } + //Assert + Assert.IsTrue(filecheck, "CycloneDx BOM not exist"); + } + + [Test, Order(3)] + public async Task ComponentCreation_AfterSuccessfulExeRun_ReturnsSuccess() + { + //Setting the httpclient + var httpClient = new HttpClient() { }; + httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue(testParameters.SW360AuthTokenType, testParameters.SW360AuthTokenValue); + string expectedcomponentType = "OSS"; + string expectedname = "beautifulsoup4"; + //url formation for retrieving component details + string url = TestConstant.Sw360ComponentApi + TestConstant.componentNameUrl + "beautifulsoup4"; + string responseBody = await httpClient.GetStringAsync(url); //GET request + var responseData = JsonConvert.DeserializeObject(responseBody); + string name = responseData.Embedded.Sw360components[0].Name; + string href = responseData.Embedded.Sw360components[0].Links.Self.Href; + string hrefresponse = await httpClient.GetStringAsync(href);//GET request + var hrefData = JsonConvert.DeserializeObject(hrefresponse); + string componentType = hrefData.ComponentType; + + //Assert + Assert.AreEqual(expectedname, name, "Test if the component name is correct"); + Assert.AreEqual(expectedcomponentType, componentType, "Test if the component version is correct"); + } + + [Test, Order(4)] + public async Task ReleaseCreation__AfterSuccessfulExeRun_ReturnsClearingStateAsNewClearing() + { + //Setting the httpclient + var httpClient = new HttpClient() { }; + httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue(testParameters.SW360AuthTokenType, testParameters.SW360AuthTokenValue); + string expectedname = "beautifulsoup4"; + string expectedversion = "4.11.1"; + string expectedclearingState = "NEW_CLEARING"; + string expecteddownloadurl = "https://files.pythonhosted.org/packages/e8/b0/cd2b968000577ec5ce6c741a54d846dfa402372369b8b6861720aa9ecea7/beautifulsoup4-4.11.1.tar.gz"; + string expectedexternalid = "pkg:pypi/beautifulsoup4@4.11.1"; + //url formation for retrieving component details + string url = TestConstant.Sw360ReleaseApi + TestConstant.componentNameUrl + "beautifulsoup4"; + string responseBody = await httpClient.GetStringAsync(url);//GET method + var responseData = JsonConvert.DeserializeObject(responseBody); + string urlofreleaseid = responseData.Embedded.Sw360Releases[0].Links.Self.Href; + string responseForRelease = await httpClient.GetStringAsync(urlofreleaseid);//GET method for fetching the release details + var responseDataForRelease = JsonConvert.DeserializeObject(responseForRelease); + + string name = responseDataForRelease.Name; + string version = responseDataForRelease.Version; + string downloadurl = responseDataForRelease.SourceDownloadurl; + string clearingState = responseDataForRelease.ClearingState; + string externalid = responseDataForRelease.ExternalIds.Package_Url; + + //Assert + Assert.AreEqual(expectedname, name, "Test Project Name"); + Assert.AreEqual(expectedversion, version, "Test Project Version"); + Assert.AreEqual(expecteddownloadurl, downloadurl, "Test download Url of Entity Framework"); + Assert.AreEqual(expectedclearingState, clearingState, "Test component clearing state"); + Assert.AreEqual(expectedexternalid, externalid, "Test component external id"); + } + + private string CCTComparisonBomTestFile { get; set; } + private string OutFolder { get; set; } + } +} \ No newline at end of file diff --git a/src/SW360IntegrationTest/Python/PackageIdentifierInitialPython.cs b/src/SW360IntegrationTest/Python/PackageIdentifierInitialPython.cs new file mode 100644 index 00000000..17b9d6e1 --- /dev/null +++ b/src/SW360IntegrationTest/Python/PackageIdentifierInitialPython.cs @@ -0,0 +1,95 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (C) Siemens AG 2023. All rights reserved. +// +//MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using NUnit.Framework; +using System.IO; +using TestUtilities; + +namespace SW360IntegrationTest.Python +{ + [TestFixture, Order(23)] + public class PackageIdentifierInitialPython + { + private string CCTLocalBomTestFile { get; set; } + private string OutFolder { get; set; } + TestParamNuget testParameters; + + [SetUp] + public void Setup() + { + OutFolder = TestHelper.OutFolder; + + CCTLocalBomTestFile = OutFolder + @"..\..\..\src\SW360IntegrationTest\PackageIdentifierTestFiles\Python\CCTLocalBOMPythonInitial.json"; + + if (!Directory.Exists(OutFolder + @"\..\BOMs")) + { + Directory.CreateDirectory(OutFolder + @"\..\BOMs"); + } + testParameters = new TestParamNuget(); + } + + [Test, Order(1)] + public void RunBOMCreatorexe_ProvidedPackageJsonFilePath_ReturnsSuccess() + { + string packagejsonPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\SystemTest1stIterationData\Python"; + string bomPath = OutFolder + @"\..\BOMs"; + + // Test BOM Creator ran with exit code 0 + Assert.AreEqual(0, TestHelper.RunBOMCreatorExe(new string[]{ + TestConstant.PackageFilePath, packagejsonPath, + TestConstant.BomFolderPath, bomPath, + TestConstant.Sw360Token, testParameters.SW360AuthTokenValue, + TestConstant.SW360AuthTokenType, testParameters.SW360AuthTokenType, + TestConstant.SW360URL, testParameters.SW360URL, + TestConstant.SW360ProjectID, testParameters.SW360ProjectID, + TestConstant.SW360ProjectName, testParameters.SW360ProjectName, + TestConstant.JFrogApiURL, testParameters.JfrogApi, + TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, + TestConstant.ProjectType,"PYTHON", + TestConstant.Mode,""}), + "Test to run Package Identifier EXE execution"); + } + + [Test, Order(2)] + public void LocalBOMCreation_AfterSuccessfulExeRun_ReturnsSuccess() + { + bool fileExist = false; + + // Expected + ComponentJsonParsor expected = new ComponentJsonParsor(); + expected.Read(CCTLocalBomTestFile); + + // Actual + string generatedBOM = OutFolder + $"\\..\\BOMs\\{testParameters.SW360ProjectName}_Bom.cdx.json"; + if (File.Exists(generatedBOM)) + { + fileExist = true; + + ComponentJsonParsor actual = new ComponentJsonParsor(); + actual.Read(generatedBOM); + + foreach (var item in expected.Components) + { + foreach (var i in actual.Components) + { + if ((i.Name == item.Name) && (i.Version == item.Version)) + { + Component component = i; + Assert.AreEqual(item.Name, component.Name); + Assert.AreEqual(item.Version, component.Version); + Assert.AreEqual(item.Purl, component.Purl); + Assert.AreEqual(item.BomRef, component.BomRef); + } + } + } + } + //Assert + Assert.IsTrue(fileExist, "Test to BOM file present"); + } + } +} diff --git a/src/SW360IntegrationTest/SW360IntegrationTest.csproj b/src/SW360IntegrationTest/SW360IntegrationTest.csproj index 07f7c7a2..dfdb730b 100644 --- a/src/SW360IntegrationTest/SW360IntegrationTest.csproj +++ b/src/SW360IntegrationTest/SW360IntegrationTest.csproj @@ -72,6 +72,12 @@ Always + + Always + + + Always + From 1777ef51f224bee148ae00652d83039ed2c19909 Mon Sep 17 00:00:00 2001 From: sumanthkb44 <84563853+sumanthkb44@users.noreply.github.com> Date: Thu, 14 Sep 2023 11:43:10 +0530 Subject: [PATCH 09/18] Update compile.yml --- .github/workflows/compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 0ef64c47..77416f94 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -115,7 +115,7 @@ jobs: DOCKERDEVARTIFACTORY: ${{ secrets.DOCKERDEVARTIFACTORY }} - name: Test - if: ${{ false }} # disable for now + #if: ${{ false }} # disable for now run: | $TestProjects = Get-ChildItem -Path *test*.csproj -Recurse -exclude TestUtilities.csproj,UnitTestUtilities.csproj Write-Host "**************************The test projects considered for execution: $TestProjects ******************************" From 0fe7967799657c54b6846e59380bddc0f835e35e Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 14 Sep 2023 15:44:34 +0530 Subject: [PATCH 10/18] Update --- src/LCT.PackageIdentifier/NugetProcessor.cs | 2 +- src/LCT.PackageIdentifier/PythonProcessor.cs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 0368462f..3a800b48 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -390,7 +390,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List listofComponents = new(); ParseInputFiles(appSettings, filepath, listofComponents); ConvertToCycloneDXModel(listComponentForBOM, listofComponents, dependencies); - if (bom.Dependencies == null) + if (bom.Dependencies == null || bom.Dependencies.Count == 0) { bom.Dependencies = dependencies; dependencies = new List(); diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index ddf69ad7..d28fd8f6 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -307,7 +307,7 @@ private static List PoetrySetOfCmds(string SourceFilePath, List Date: Thu, 21 Sep 2023 18:33:09 +0530 Subject: [PATCH 11/18] JFrog changes --- .../PackageUploadHelper.cs | 88 ++++++++++++++++--- src/ArtifactoryUploader/PackageUploader.cs | 4 +- src/ArtifactoryUploader/Program.cs | 21 +++++ src/LCT.APICommunications/ApiConstant.cs | 1 + .../ArtifactoryUploader.cs | 19 ++-- .../Model/ComponentsToArtifactory.cs | 2 + .../PythonJfrogAPICommunication.cs | 17 ++++ src/LCT.Common/CommonAppSettings.cs | 1 + src/LCT.Common/CommonHelper.cs | 2 +- src/LCT.PackageIdentifier/PythonProcessor.cs | 5 +- src/LCT.Services/JFrogService.cs | 2 +- 11 files changed, 134 insertions(+), 28 deletions(-) diff --git a/src/ArtifactoryUploader/PackageUploadHelper.cs b/src/ArtifactoryUploader/PackageUploadHelper.cs index 6800aacb..01861210 100644 --- a/src/ArtifactoryUploader/PackageUploadHelper.cs +++ b/src/ArtifactoryUploader/PackageUploadHelper.cs @@ -8,15 +8,19 @@ using CycloneDX.Models; using LCT.APICommunications; using LCT.APICommunications.Model; +using LCT.APICommunications.Model.AQL; using LCT.ArtifactoryUploader.Model; using LCT.Common; using LCT.Common.Constants; +using LCT.Services; +using LCT.Services.Interface; using log4net; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; using System.Net; using System.Net.Http; using System.Reflection; @@ -30,6 +34,8 @@ namespace LCT.ArtifactoryUploader public static class PackageUploadHelper { static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + public static IJFrogService jFrogService { get; set; } + private static List aqlResultList = new(); private static bool SetWarningCode; public static Bom GetComponentListFromComparisonBOM(string comparisionBomFilePath) @@ -58,28 +64,29 @@ public static Bom GetComponentListFromComparisonBOM(string comparisionBomFilePat return componentsToBoms; } - public static List GetComponentsToBeUploadedToArtifactory(List comparisonBomData, CommonAppSettings appSettings) + public async static Task> GetComponentsToBeUploadedToArtifactory(List comparisonBomData, CommonAppSettings appSettings) { Logger.Debug("Starting GetComponentsToBeUploadedToArtifactory() method"); List componentsToBeUploaded = new List(); foreach (var item in comparisonBomData) { - - if (item.Properties.Any(p => p.Name == Dataconstant.Cdx_ClearingState && p.Value.ToUpperInvariant() == "APPROVED")) + if (item.Properties.Exists(p => p.Name == Dataconstant.Cdx_ClearingState && p.Value.ToUpperInvariant() == "APPROVED")) { - + AqlResult aqlResult = await GetSrcRepoDetailsForPyPiPackages(item, appSettings); ComponentsToArtifactory components = new ComponentsToArtifactory() { Name = !string.IsNullOrEmpty(item.Group) ? $"{item.Group}/{item.Name}" : item.Name, PackageName = item.Name, Version = item.Version, ComponentType = GetComponentType(item), - SrcRepoName = item.Properties[1].Value, + SrcRepoName = item.Properties.Find(s => s.Name == Dataconstant.Cdx_ArtifactoryRepoUrl).Value, DestRepoName = GetDestinationRepo(item, appSettings), ApiKey = appSettings.ArtifactoryUploadApiKey, Email = appSettings.ArtifactoryUploadUser, - JfrogApi = appSettings.JFrogApi + JfrogApi = appSettings.JFrogApi, + SrcRepoPathWithFullName = aqlResult != null ? aqlResult.Repo + "/" + aqlResult.Path + "/" + aqlResult.Name : string.Empty, + PypiCompName = aqlResult != null ? aqlResult.Name : string.Empty }; components.PackageInfoApiUrl = GetPackageInfoURL(components); components.CopyPackageApiUrl = GetCopyURL(components); @@ -114,6 +121,11 @@ private static string GetCopyURL(ComponentsToArtifactory component) url = $"{component.JfrogApi}{ApiConstant.CopyPackageApi}{component.SrcRepoName}/{component.Name}/{component.Version}" + $"?to=/{component.DestRepoName}/{component.Name}/{component.Version}"; } + else if (component.ComponentType == "PYTHON") + { + url = $"{component.JfrogApi}{ApiConstant.CopyPackageApi}{component.SrcRepoPathWithFullName}" + + $"?to=/{component.DestRepoName}/{component.PypiCompName}"; + } else { // Do nothing @@ -136,6 +148,10 @@ private static string GetPackageInfoURL(ComponentsToArtifactory component) { url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.Name}/{component.Version}"; } + else if (component.ComponentType == "PYTHON") + { + url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoPathWithFullName}"; + } else { // Do nothing @@ -157,6 +173,10 @@ private static string GetDestinationRepo(Component item, CommonAppSettings appSe { return appSettings.JfrogMavenDestRepoName; } + else if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase)) + { + return appSettings.JfrogPythonDestRepoName; + } else { // Do nothing @@ -179,6 +199,10 @@ private static string GetComponentType(Component item) { return "MAVEN"; } + else if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase)) + { + return "PYTHON"; + } else { // Do nothing @@ -186,12 +210,28 @@ private static string GetComponentType(Component item) return string.Empty; } - public static async Task UploadingThePackages(List componentsToUpload,int timeout) - { + private async static Task GetSrcRepoDetailsForPyPiPackages(Component item, CommonAppSettings appSettings) + { + if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase) && aqlResultList.Count == 0) + { + // get the component list from Jfrog for given repo + aqlResultList = await GetListOfComponentsFromRepo(appSettings.Python?.JfrogPythonRepoList, jFrogService); + } + + if (aqlResultList.Count > 0) + { + return GetArtifactoryRepoName(aqlResultList, item); + } + + return null; + } + + public static async Task UploadingThePackages(List componentsToUpload, int timeout) + { Logger.Debug("Starting UploadingThePackages() method"); foreach (var item in componentsToUpload) { - await PackageUploadToArtifactory(PackageUploader.uploaderKpiData, item,timeout); + await PackageUploadToArtifactory(PackageUploader.uploaderKpiData, item, timeout); } if (SetWarningCode) @@ -204,13 +244,13 @@ public static async Task UploadingThePackages(List comp Program.UploaderStopWatch?.Stop(); } - private static async Task PackageUploadToArtifactory(UploaderKpiData uploaderKpiData, ComponentsToArtifactory item,int timeout) + private static async Task PackageUploadToArtifactory(UploaderKpiData uploaderKpiData, ComponentsToArtifactory item, int timeout) { if (!(item.SrcRepoName.Contains("siparty"))) { if (!(item.SrcRepoName.Contains("Not Found in JFrog"))) { - HttpResponseMessage responseMessage = await ArtfactoryUploader.UploadPackageToRepo(item,timeout); + HttpResponseMessage responseMessage = await ArtfactoryUploader.UploadPackageToRepo(item, timeout); if (responseMessage.StatusCode == HttpStatusCode.OK) { uploaderKpiData.PackagesUploadedToJfrog++; @@ -234,12 +274,9 @@ private static async Task PackageUploadToArtifactory(UploaderKpiData uploaderKpi } else { - uploaderKpiData.PackagesNotExistingInRemoteCache++; Logger.Warn($"Package {item.Name}-{item.Version} is not found in jfrog"); } - - } else { @@ -280,6 +317,29 @@ public static void WriteCreatorKpiDataToConsole(UploaderKpiData uploaderKpiData) CommonHelper.WriteToConsoleTable(printList, printTimingList); } + public static async Task> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService) + { + if (repoList != null && repoList.Length > 0) + { + foreach (var repo in repoList) + { + var test = await jFrogService.GetInternalComponentDataByRepo(repo) ?? new List(); + aqlResultList.AddRange(test); + } + } + + return aqlResultList; + } + + private static AqlResult GetArtifactoryRepoName(List aqlResultList, Component component) + { + string jfrogcomponentName = $"{component.Name}-{component.Version}"; + + AqlResult repoName = aqlResultList.Find(x => x.Name.Contains(jfrogcomponentName, StringComparison.OrdinalIgnoreCase)); + + return repoName; + } + } } diff --git a/src/ArtifactoryUploader/PackageUploader.cs b/src/ArtifactoryUploader/PackageUploader.cs index f8dc9fc6..170ae9d6 100644 --- a/src/ArtifactoryUploader/PackageUploader.cs +++ b/src/ArtifactoryUploader/PackageUploader.cs @@ -33,11 +33,11 @@ public static async Task UploadPackageToArtifactory(CommonAppSettings appSetting uploaderKpiData.ComponentInComparisonBOM = m_ComponentsInBOM.Components.Count; - List m_ComponentsToBeUploaded = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(m_ComponentsInBOM.Components, appSettings); + List m_ComponentsToBeUploaded = await PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(m_ComponentsInBOM.Components, appSettings); //Uploading the component to artifactory uploaderKpiData.PackagesToBeUploaded = m_ComponentsToBeUploaded.Count; - await PackageUploadHelper.UploadingThePackages(m_ComponentsToBeUploaded,appSettings.TimeOut); + await PackageUploadHelper.UploadingThePackages(m_ComponentsToBeUploaded, appSettings.TimeOut); // write kpi info to console table PackageUploadHelper.WriteCreatorKpiDataToConsole(uploaderKpiData); diff --git a/src/ArtifactoryUploader/Program.cs b/src/ArtifactoryUploader/Program.cs index 2da69621..b8d573f3 100644 --- a/src/ArtifactoryUploader/Program.cs +++ b/src/ArtifactoryUploader/Program.cs @@ -5,11 +5,16 @@ // -------------------------------------------------------------------------------------------------------------------- using LCT.APICommunications; +using LCT.APICommunications.Interfaces; using LCT.APICommunications.Model; using LCT.ArtifactoryUploader; using LCT.Common; using LCT.Common.Constants; using LCT.Common.Interface; +using LCT.Facade.Interfaces; +using LCT.Facade; +using LCT.Services.Interface; +using LCT.Services; using log4net; using log4net.Core; using System; @@ -65,7 +70,9 @@ static async Task Main(string[] args) await artifactoryValidator.ValidateArtifactoryCredentials(appSettings); //Uploading Package to artifactory + PackageUploadHelper.jFrogService = GetJfrogService(appSettings); await PackageUploader.UploadPackageToArtifactory(appSettings); + Logger.Logger.Log(null, Level.Notice, $"End of Artifactory Uploader execution : {DateTime.Now}\n", null); } @@ -93,5 +100,19 @@ private static string InitiateLogger(CommonAppSettings appSettings) return FolderPath; } + + private static IJFrogService GetJfrogService(CommonAppSettings appSettings) + { + ArtifactoryCredentials artifactoryUpload = new ArtifactoryCredentials() + { + ApiKey = appSettings.ArtifactoryUploadApiKey + }; + IJfrogAqlApiCommunication jfrogAqlApiCommunication = + new JfrogAqlApiCommunication(appSettings.JFrogApi, artifactoryUpload, appSettings.TimeOut); + IJfrogAqlApiCommunicationFacade jFrogApiCommunicationFacade = + new JfrogAqlApiCommunicationFacade(jfrogAqlApiCommunication); + IJFrogService jFrogService = new JFrogService(jFrogApiCommunicationFacade); + return jFrogService; + } } } diff --git a/src/LCT.APICommunications/ApiConstant.cs b/src/LCT.APICommunications/ApiConstant.cs index b896eaa2..d004b5a8 100644 --- a/src/LCT.APICommunications/ApiConstant.cs +++ b/src/LCT.APICommunications/ApiConstant.cs @@ -32,6 +32,7 @@ public static class ApiConstant public const string NpmExtension = ".tgz"; public const string NugetExtension = ".nupkg"; public const string MavenExtension = "-sources.jar"; + public const string PythonExtension = ".whl"; public const string PackageInfoApi = "/api/storage/"; public const string CopyPackageApi = "/api/copy/"; public const string Releases = "releases"; diff --git a/src/LCT.APICommunications/ArtifactoryUploader.cs b/src/LCT.APICommunications/ArtifactoryUploader.cs index 2d737b35..b19e2855 100644 --- a/src/LCT.APICommunications/ArtifactoryUploader.cs +++ b/src/LCT.APICommunications/ArtifactoryUploader.cs @@ -16,15 +16,15 @@ namespace LCT.APICommunications { - public static class ArtfactoryUploader + public static class ArtfactoryUploader { //ConfigurationAttribute private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private static string destRepoName = Environment.GetEnvironmentVariable("JfrogDestRepoName"); private static string JfrogApi = Environment.GetEnvironmentVariable("JfrogApi"); private static string srcRepoName = Environment.GetEnvironmentVariable("JfrogSrcRepo"); - - public static async Task UploadPackageToRepo(ComponentsToArtifactory component,int timeout) + + public static async Task UploadPackageToRepo(ComponentsToArtifactory component, int timeout) { Logger.Debug("Starting UploadPackageToArtifactory method"); IJFrogApiCommunication jfrogApicommunication; @@ -37,14 +37,19 @@ public static async Task UploadPackageToRepo(ComponentsToAr ApiKey = component.ApiKey, Email = component.Email }; - if (component?.ComponentType?.ToUpperInvariant() == "MAVEN") + if (component.ComponentType?.ToUpperInvariant() == "MAVEN") + { + jfrogApicommunication = new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); + responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); + } + else if (component.ComponentType?.ToUpperInvariant() == "PYTHON") { - jfrogApicommunication = new MavenJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials,timeout); + jfrogApicommunication = new PythonJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); } else { - jfrogApicommunication = new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials,timeout); + jfrogApicommunication = new NpmJfrogApiCommunication(component.JfrogApi, component.SrcRepoName, repoCredentials, timeout); responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); } if (responseBodyJfrog.StatusCode == HttpStatusCode.NotFound) @@ -53,8 +58,6 @@ public static async Task UploadPackageToRepo(ComponentsToAr responseBodyJfrog = await jfrogApicommunication.GetPackageInfo(component); component.CopyPackageApiUrl = component.CopyPackageApiUrl.ToLower(); } - - if (responseBodyJfrog.StatusCode != HttpStatusCode.OK) { responsemessage.StatusCode = responseBodyJfrog.StatusCode; diff --git a/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs b/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs index 819b4f98..2e0292f0 100644 --- a/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs +++ b/src/LCT.APICommunications/Model/ComponentsToArtifactory.cs @@ -16,6 +16,8 @@ public class ComponentsToArtifactory public string ComponentType { get; set; } public string JfrogApi { get; set; } public string SrcRepoName { get; set; } + public string SrcRepoPathWithFullName { get; set; } + public string PypiCompName { get; set; } public string DestRepoName { get; set; } public string ApiKey { get; set; } public string Email { get; set; } diff --git a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs index 948785d2..44106160 100644 --- a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs +++ b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs @@ -6,6 +6,8 @@ // -------------------------------------------------------------------------------------------------------------------- using LCT.APICommunications.Model; +using LCT.APICommunications.Model.AQL; +using System.Collections.Generic; using System.Net.Http; using System.Threading.Tasks; @@ -52,6 +54,21 @@ public override void UpdatePackagePropertiesInJfrog(string sw360releaseUrl, stri httpClient.PutAsync(url, httpContent); } + //public async Task> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService) + //{ + // List aqlResultList = new(); + // if (repoList != null && repoList.Length > 0) + // { + // foreach (var repo in repoList) + // { + // var test = await jFrogService.GetInternalComponentDataByRepo(repo) ?? new List(); + // aqlResultList.AddRange(test); + // } + // } + + // return aqlResultList; + //} + } } diff --git a/src/LCT.Common/CommonAppSettings.cs b/src/LCT.Common/CommonAppSettings.cs index 5029c334..ac9580e9 100644 --- a/src/LCT.Common/CommonAppSettings.cs +++ b/src/LCT.Common/CommonAppSettings.cs @@ -73,6 +73,7 @@ public CommonAppSettings(IFolderAction iFolderAction) public string JfrogNpmSrcRepo { get; set; } public string JfrogNugetDestRepoName { get; set; } public string JfrogMavenDestRepoName { get; set; } + public string JfrogPythonDestRepoName { get; set; } public string JfrogNugetSrcRepo { get; set; } public string Mode { get; set; } = string.Empty; diff --git a/src/LCT.Common/CommonHelper.cs b/src/LCT.Common/CommonHelper.cs index 914dda8b..55162fe5 100644 --- a/src/LCT.Common/CommonHelper.cs +++ b/src/LCT.Common/CommonHelper.cs @@ -47,7 +47,7 @@ public static List RemoveExcludedComponents(List Component name = $"{component.Group}/{component.Name}"; } - if (name.ToLowerInvariant() == excludedcomponent[0].ToLowerInvariant() && (component.Version.ToLowerInvariant() == excludedcomponent[1].ToLowerInvariant() || excludedcomponent[1].ToLowerInvariant() == "*")) + if (name.ToLowerInvariant() == excludedcomponent[0].ToLowerInvariant() && excludedcomponent.Length > 0 && (component.Version.ToLowerInvariant() == excludedcomponent[1].ToLowerInvariant() || excludedcomponent[1].ToLowerInvariant() == "*")) { noOfExcludedComponents++; ExcludedList.Add(component); diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index d28fd8f6..0773258d 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -5,6 +5,7 @@ // -------------------------------------------------------------------------------------------------------------------- using CycloneDX.Models; +using LCT.APICommunications; using LCT.APICommunications.Model.AQL; using LCT.Common; using LCT.Common.Constants; @@ -463,7 +464,7 @@ private static bool IsInternalPythonComponent(List aqlResultList, Com string fullNameVersion = $"{fullName}-{component.Version}"; if (!fullNameVersion.Equals(jfrogcomponentName, StringComparison.OrdinalIgnoreCase) && aqlResultList.Exists( - x => x.Name.Equals(fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(FileConstant.TargzFileExtension)))) + x => x.Name.Equals(fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(ApiConstant.PythonExtension) || x.Name.EndsWith(FileConstant.TargzFileExtension)))) { return true; } @@ -510,7 +511,7 @@ private static string GetArtifactoryRepoName(List aqlResultList, Comp repoName.Equals(NotFoundInRepo, StringComparison.OrdinalIgnoreCase)) { repoName = aqlResultList.Find(x => x.Name.Contains( - fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(".whl") || x.Name.EndsWith(FileConstant.TargzFileExtension)))?.Repo ?? NotFoundInRepo; + fullNameVersion, StringComparison.OrdinalIgnoreCase) && (x.Name.EndsWith(ApiConstant.PythonExtension) || x.Name.EndsWith(FileConstant.TargzFileExtension)))?.Repo ?? NotFoundInRepo; } return repoName; diff --git a/src/LCT.Services/JFrogService.cs b/src/LCT.Services/JFrogService.cs index 9fe3c69e..cf8a60a3 100644 --- a/src/LCT.Services/JFrogService.cs +++ b/src/LCT.Services/JFrogService.cs @@ -44,7 +44,7 @@ public async Task> GetInternalComponentDataByRepo(string repoNa return new List(); } - string stringData = httpResponseMessage?.Content?.ReadAsStringAsync()?.Result ?? string.Empty; + string stringData = httpResponseMessage.Content?.ReadAsStringAsync()?.Result ?? string.Empty; var aqlResponse = JsonConvert.DeserializeObject(stringData); aqlResult = aqlResponse?.Results ?? new List(); } From cc1efe8f607c2648a7034458a90ffadf2392d35a Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Fri, 22 Sep 2023 12:58:26 +0530 Subject: [PATCH 12/18] Update --- .../PythonJfrogAPICommunication.cs | 16 ---------------- .../appSettingsSW360IntegrationTest.json | 12 ++++++------ src/UnitTestUtilities/appsettingsUnitTest.json | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs index 44106160..47e4b25b 100644 --- a/src/LCT.APICommunications/PythonJfrogAPICommunication.cs +++ b/src/LCT.APICommunications/PythonJfrogAPICommunication.cs @@ -53,22 +53,6 @@ public override void UpdatePackagePropertiesInJfrog(string sw360releaseUrl, stri $"properties=sw360url={sw360releaseUrl}"; httpClient.PutAsync(url, httpContent); } - - //public async Task> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService) - //{ - // List aqlResultList = new(); - // if (repoList != null && repoList.Length > 0) - // { - // foreach (var repo in repoList) - // { - // var test = await jFrogService.GetInternalComponentDataByRepo(repo) ?? new List(); - // aqlResultList.AddRange(test); - // } - // } - - // return aqlResultList; - //} - } } diff --git a/src/TestUtilities/appSettingsSW360IntegrationTest.json b/src/TestUtilities/appSettingsSW360IntegrationTest.json index 5ea55547..3ad58a06 100644 --- a/src/TestUtilities/appSettingsSW360IntegrationTest.json +++ b/src/TestUtilities/appSettingsSW360IntegrationTest.json @@ -1,15 +1,15 @@ { - "FossologyURL": "", + "FossologyURL": "http://md2pdvnc.ad001.siemens.net:8096", "RemoveDevDependency": true, "SW360AuthTokenType": "Token", "SW360ProjectID": "0d0e23f6bccb4072be91b5a3462414ad", "SW360ProjectName": "Test", - "SW360URL": "", + "SW360URL": "http://md2pdvnc.ad001.siemens.net:8095", "JfrogNugetDestRepoName": "nuget-test", "JfrogNpmDestRepoName": "npm-test", "JfrogMavenDestRepoName": "maven-test", - "ArtifactoryUploadUser": "", - "ArtifactoryUploadApiKey": "", - "JfrogApi": "", - "SW360AuthTokenValue": "" + "ArtifactoryUploadApiKey": "AKCp8nyNrX7MxdivMy7ka2CwdLfQERMcbJ4Y6icHoQJQA6ZMjYJk4nLTrWs5TCeWXmzz8ya94", + "ArtifactoryUploadUser": "karthika.g@siemens.com", + "JfrogApi": "https://siemens.jfrog.io/artifactory", + "SW360AuthTokenValue": "mlhPtOTmz7cAG7deyyDh" } diff --git a/src/UnitTestUtilities/appsettingsUnitTest.json b/src/UnitTestUtilities/appsettingsUnitTest.json index 4922a603..df660243 100644 --- a/src/UnitTestUtilities/appsettingsUnitTest.json +++ b/src/UnitTestUtilities/appsettingsUnitTest.json @@ -1,5 +1,5 @@ { "SW360URL": "http://localhost:8090", "FossologyURL": "http://localhost:8091", - "JFrogURL": "" + "JFrogURL": "https://siemens.jfrog.io/artifactory" } \ No newline at end of file From b35f084f46b989b07126ff22f2cd280764102848 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Fri, 22 Sep 2023 15:34:46 +0530 Subject: [PATCH 13/18] IT Added --- .../PythonComparisonBOM.json | 252 ++++++++++++++++++ .../Python/ArtifactoryUploaderPython.cs | 57 ++++ 2 files changed, 309 insertions(+) create mode 100644 TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json create mode 100644 src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs diff --git a/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json new file mode 100644 index 00000000..bad2c2d4 --- /dev/null +++ b/TestFiles/IntegrationTestFiles/ArtifactoryUploaderTestData/PythonComparisonBOM.json @@ -0,0 +1,252 @@ +{ + "BomFormat": "CycloneDX", + "SpecVersion": 4, + "SpecVersionString": "1.4", + "SerialNumber": null, + "Version": null, + "Metadata": { + "Tools": [ + { + "Vendor": "Siemens AG", + "Name": "Clearing Automation Tool", + "Version": "3.1.0", + "Hashes": null + } + ], + "Authors": null, + "Component": null, + "Manufacture": null, + "Supplier": null + }, + "Components": [ + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/html5lib@1.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "html5lib", + "Version": "1.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/html5lib@1.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "org1-pythonhosted-pypi-remote-cache" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Approved" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/f697515c180646c3b8b9b70ce6f2a2d8" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/attrs@22.1.0", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "attrs", + "Version": "22.1.0", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/attrs@22.1.0", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "Not Found in JFrogRepo" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Approved" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/37fcf2d0725a46b8b777c849111e9be4" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/beautifulsoup4@4.11.1", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "beautifulsoup4", + "Version": "4.11.1", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/beautifulsoup4@4.11.1", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "org1-pythonhosted-pypi-remote-cache" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Approved" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/4b92d5ac32f84d43a82f2c2c57771e6d" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + }, + { + "Type": 0, + "MimeType": null, + "BomRef": "pkg:pypi/cachy@0.3.0", + "Supplier": null, + "Author": null, + "Publisher": null, + "Group": null, + "Name": "cachy", + "Version": "0.3.0", + "Description": "", + "Scope": null, + "Hashes": null, + "Licenses": null, + "Copyright": null, + "Cpe": null, + "Purl": "pkg:pypi/cachy@0.3.0", + "Swid": null, + "Modified": null, + "Pedigree": null, + "Components": null, + "Properties": [ + { + "Name": "internal:siemens:clearing:development", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:identifier-type", + "Value": "ManuallyAdded" + }, + { + "Name": "internal:siemens:clearing:is-internal", + "Value": "false" + }, + { + "Name": "internal:siemens:clearing:repo-url", + "Value": "org1-pythonhosted-pypi-remote-cache" + }, + { + "Name": "internal:siemens:clearing:project-type", + "Value": "PYTHON" + }, + { + "Name": "internal:siemens:clearing:clearing-state", + "Value": "Approved" + }, + { + "Name": "internal:siemens:clearing:sw360:release-url", + "Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/cece81ace4b14374af2432067bca5fb2" + }, + { + "Name": "internal:siemens:clearing:fossology:url", + "Value": null + } + ], + "Evidence": null + } + ], + "Compositions": null +} \ No newline at end of file diff --git a/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs b/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs new file mode 100644 index 00000000..5a10e1e7 --- /dev/null +++ b/src/SW360IntegrationTest/Python/ArtifactoryUploaderPython.cs @@ -0,0 +1,57 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (C) Siemens AG 2023. All rights reserved. +// +//MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using NUnit.Framework; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using TestUtilities; + +namespace SW360IntegrationTest.Python +{ + [TestFixture, Order(25)] + public class ArtifactoryUploaderPython + { + private string OutFolder { get; set; } + private static readonly TestParam testParameters = new TestParam(); + [Test, Order(1)] + public void TestArtifactoryUploaderexe() + { + OutFolder = TestHelper.OutFolder; + string comparisonBOMPath = OutFolder + @"\..\..\TestFiles\IntegrationTestFiles\ArtifactoryUploaderTestData\PythonComparisonBOM.json"; + + int result = TestHelper.RunArtifactoryUploaderExe(new string[]{ + TestConstant.BomFilePath, comparisonBOMPath, + TestConstant.ArtifactoryUser, testParameters.ArtifactoryUploadUser, + TestConstant.ArtifactoryKey, testParameters.ArtifactoryUploadApiKey, + TestConstant.JfrogNPMDestRepoName,testParameters.DestinationRepoName, + TestConstant.JFrogApiURL,testParameters.JfrogApi + }); + + // Test BOM Creator ran with exit code 0 or 2 (Warning) + Assert.IsTrue(result == 0 || result == 2, + "Test to run Artifactory Uploader EXE execution"); + } + + [Test, Order(2)] + public void ComponentUpload_IsFailure() + { + HttpClient httpClient = new HttpClient(); + httpClient.DefaultRequestHeaders.Add(TestConstant.JFrog_API_Header, testParameters.ArtifactoryUploadApiKey); + httpClient.DefaultRequestHeaders.Add(TestConstant.Email, testParameters.ArtifactoryUploadUser); + + // Act + string url = $"{TestConstant.JfrogApi}/pypi-test/cachy222-0.3.0-py2.py3-none-any.whl"; + HttpResponseMessage responseBody = httpClient.GetAsync(url).Result; + + // Assert + Assert.That(HttpStatusCode.NotFound, Is.EqualTo(responseBody.StatusCode), "Returns Failure status code"); + } + } +} From 6bdf838b1865e7709207310c461b31a0e6f693d3 Mon Sep 17 00:00:00 2001 From: MadanReddyK <125262006+MadanReddyK@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:16:11 +0530 Subject: [PATCH 14/18] Update compile.yml for testing the build --- .github/workflows/compile.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 77416f94..693028fa 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -49,8 +49,7 @@ jobs: Write-Host "Filename: '$fileName'" - - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.0.2 + - name: update appSettingsSW360IntegrationTest.json run: ./Scripts/json_update.ps1 -JsonFile 'src\TestUtilities\appSettingsSW360IntegrationTest.json' From 5ddc3426b7a93076259263460091183aafdb0816 Mon Sep 17 00:00:00 2001 From: MadanReddyK <125262006+MadanReddyK@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:19:34 +0530 Subject: [PATCH 15/18] Update compile.yml --- .github/workflows/compile.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 693028fa..89b79a1b 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -48,7 +48,8 @@ jobs: echo "SemVerMMP: v$($env:GitVersion_MajorMinorPatch)" Write-Host "Filename: '$fileName'" - + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.3.1 - name: update appSettingsSW360IntegrationTest.json From bad57d50ee434de768d8a3607e26d94df10f75aa Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Mon, 25 Sep 2023 15:22:13 +0530 Subject: [PATCH 16/18] UT issues --- .../PackageUploadHelperTest.cs | 20 ++++++++++--------- .../PackageUploadHelper.cs | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs index f21526b4..8042b897 100644 --- a/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs +++ b/src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs @@ -14,6 +14,7 @@ using System.Collections.Generic; using System.IO; using UnitTestUtilities; +using System.Threading.Tasks; namespace AritfactoryUploader.UTest { @@ -56,7 +57,7 @@ public void GetComponentListFromComparisonBOM_GivenInvalidfile_ReturnsException( [Test] - public void GetComponentsToBeUploadedToArtifactory_GivenFewApprovedComponentList_ReturnsUploadList() + public async Task GetComponentsToBeUploadedToArtifactory_GivenFewApprovedComponentList_ReturnsUploadList() { //Arrange List componentLists = GetComponentList(); @@ -73,14 +74,14 @@ public void GetComponentsToBeUploadedToArtifactory_GivenFewApprovedComponentList }; //Act - List uploadList = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); + List uploadList = await PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); // Assert Assert.That(3, Is.EqualTo(uploadList.Count), "Checks for 2 no of components to upload"); } [Test] - public void GetComponentsToBeUploadedToArtifactory_GivenAllApprovedComponentList_ReturnsUploadList() + public async Task GetComponentsToBeUploadedToArtifactory_GivenAllApprovedComponentList_ReturnsUploadList() { //Arrange List componentLists = GetComponentList(); @@ -101,18 +102,18 @@ public void GetComponentsToBeUploadedToArtifactory_GivenAllApprovedComponentList JfrogNpmDestRepoName = "npm-test", JfrogNpmSrcRepo = "remote-cache", JFrogApi = UTParams.JFrogURL, - LogFolderPath= outFolder + LogFolderPath = outFolder }; - string LogfolderPath = appSettings.LogFolderPath; //Act - List uploadList = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); + List uploadList = await PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); // Assert Assert.That(4, Is.EqualTo(uploadList.Count), "Checks for 3 no of components to upload"); } + [Test] - public void GetComponentsToBeUploadedToArtifactory_GivenNotApprovedComponentList_ReturnsUploadList() + public async Task GetComponentsToBeUploadedToArtifactory_GivenNotApprovedComponentList_ReturnsUploadList() { //Arrange List componentLists = GetComponentList(); @@ -129,13 +130,14 @@ public void GetComponentsToBeUploadedToArtifactory_GivenNotApprovedComponentList JfrogNugetSrcRepo = "remote-cache", JFrogApi = UTParams.JFrogURL }; - + //Act - List uploadList = PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); + List uploadList =await PackageUploadHelper.GetComponentsToBeUploadedToArtifactory(componentLists, appSettings); // Assert Assert.That(0, Is.EqualTo(uploadList.Count), "Checks for components to upload to be zero"); } + private static List GetComponentList() { List componentLists = new List(); diff --git a/src/ArtifactoryUploader/PackageUploadHelper.cs b/src/ArtifactoryUploader/PackageUploadHelper.cs index 01861210..7adbe621 100644 --- a/src/ArtifactoryUploader/PackageUploadHelper.cs +++ b/src/ArtifactoryUploader/PackageUploadHelper.cs @@ -80,7 +80,7 @@ public async static Task> GetComponentsToBeUploade PackageName = item.Name, Version = item.Version, ComponentType = GetComponentType(item), - SrcRepoName = item.Properties.Find(s => s.Name == Dataconstant.Cdx_ArtifactoryRepoUrl).Value, + SrcRepoName = item.Properties.Find(s => s.Name == Dataconstant.Cdx_ArtifactoryRepoUrl)?.Value, DestRepoName = GetDestinationRepo(item, appSettings), ApiKey = appSettings.ArtifactoryUploadApiKey, Email = appSettings.ArtifactoryUploadUser, From 8a2686455a73cdb8c1fa9e11ae23c6e97dcec7e2 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Wed, 27 Sep 2023 16:11:28 +0530 Subject: [PATCH 17/18] Sonar Chnages --- .../NugetDevDependencyParser.cs | 53 +++++++++++-------- src/LCT.PackageIdentifier/NugetProcessor.cs | 2 +- src/LCT.PackageIdentifier/PythonProcessor.cs | 52 ++++++++++-------- src/LCT.SW360PackageCreator/URLHelper.cs | 4 +- 4 files changed, 63 insertions(+), 48 deletions(-) diff --git a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs index 55eaee8e..4ba9c63a 100644 --- a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs +++ b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs @@ -44,7 +44,7 @@ public static NugetDevDependencyParser Instance } } - public List Parse(string configFile) + public static List Parse(string configFile) { List containerList = new(); @@ -132,27 +132,8 @@ private static void ParseJsonFile(string filePath, Container container) } else { - string csprojFilePath = ""; - string dirName = Path.GetDirectoryName(filePath); - if (dirName.Contains("obj")) - { - dirName = dirName.Replace("obj", ""); - string[] filePaths = Directory.GetFiles(dirName, "*.csproj"); - csprojFilePath = filePaths.Length > 0 ? filePaths[0] : ""; - } - if(!string.IsNullOrEmpty(csprojFilePath) && File.Exists(csprojFilePath)) - { - Logger.Debug($"ParseJsonFile():Linux Asset FileName: " + csprojFilePath); - isTestProject = IsTestProject(csprojFilePath); - container.Name = Path.GetFileName(csprojFilePath); - Logger.Debug($"ParseJsonFile():Linux Asset File: IsTestProject: " + isTestProject); - } - else - { - Logger.Debug($"ParseJsonFile():Linux Asset FileName Not Found!! "); - isTestProject = false; - container.Name = Path.GetFileName(filePath); - } + //when it's running as container + isTestProject = ParseJsonInContainer(filePath, ref container); } if (isTestProject) @@ -179,6 +160,34 @@ private static void ParseJsonFile(string filePath, Container container) } } + private static bool ParseJsonInContainer(string filePath, ref Container container) + { + bool isTestProject; + string csprojFilePath = ""; + string dirName = Path.GetDirectoryName(filePath); + if (dirName.Contains("obj")) + { + dirName = dirName.Replace("obj", ""); + string[] filePaths = Directory.GetFiles(dirName, "*.csproj"); + csprojFilePath = filePaths.Length > 0 ? filePaths[0] : ""; + } + if (!string.IsNullOrEmpty(csprojFilePath) && File.Exists(csprojFilePath)) + { + Logger.Debug($"ParseJsonFile():Linux Asset FileName: " + csprojFilePath); + isTestProject = IsTestProject(csprojFilePath); + container.Name = Path.GetFileName(csprojFilePath); + Logger.Debug($"ParseJsonFile():Linux Asset File: IsTestProject: " + isTestProject); + } + else + { + Logger.Debug($"ParseJsonFile():Linux Asset FileName Not Found!! "); + isTestProject = false; + container.Name = Path.GetFileName(filePath); + } + + return isTestProject; + } + private static void ParseLibrary(LockFileTargetLibrary library, bool isTestProject, IDictionary components, LockFile assetFile) { if (library.Type.Equals("project", StringComparison.InvariantCultureIgnoreCase)) diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 3a800b48..f4ba45ae 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -633,7 +633,7 @@ private static string ReferenceTagDetailsForPackageReference(XmlNode childNode, private static List ParseAssetFile(string configFile) { NugetDevDependencyParser nugetDevDependencyParser = NugetDevDependencyParser.Instance; - List containers = nugetDevDependencyParser.Parse(configFile); + List containers = NugetDevDependencyParser.Parse(configFile); return ConvertContainerAsNugetPackage(containers, configFile); } diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index 0773258d..b793baba 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -6,6 +6,7 @@ using CycloneDX.Models; using LCT.APICommunications; +using LCT.APICommunications.Model; using LCT.APICommunications.Model.AQL; using LCT.Common; using LCT.Common.Constants; @@ -88,7 +89,7 @@ public static List ExtractDetailsForPoetryLockfile(string filePat PythonPackages = PoetrySetOfCmds(filePath, dependencies); return PythonPackages; } - + private List ExtractDetailsFromJson(string filePath, CommonAppSettings appSettings, ref List dependencies) { List PythonPackages = new List(); @@ -328,30 +329,10 @@ private static Dependency GetDependenciesDetails(Result result, PythonPackage ma { var details = result.StdOut; List lines = details.Split(Environment.NewLine).ToList(); - bool addDependencies = false; - List dependencyList = new List(); - - foreach (string line in lines) - { - if (line == "dependencies") - { - addDependencies = true; - continue; - } - - if (addDependencies && !string.IsNullOrEmpty(line)) - { - string comp = line; - comp = comp.Replace(" - ", ""); - dependencyList.Add(comp.Split(" ")[0]); - } - - if (string.IsNullOrEmpty(line)) - addDependencies = false; - } + List dependencyList = GetDetailsFromLines(lines); dependency = GetDependencyMappings(mainComp, dependencyList, AllComps); } - else if(result != null && result.StdOut.Contains("name")) + else if (result != null && result.StdOut.Contains("name")) { return new Dependency() { @@ -366,6 +347,31 @@ private static Dependency GetDependenciesDetails(Result result, PythonPackage ma return dependency; } + private static List GetDetailsFromLines(List lines) + { + bool addDependencies = false; + List dependencyList = new List(); + foreach (string line in lines) + { + if (line == "dependencies") + { + addDependencies = true; + continue; + } + + if (addDependencies && !string.IsNullOrEmpty(line)) + { + string comp = line; + comp = comp.Replace(" - ", ""); + dependencyList.Add(comp.Split(" ")[0]); + } + + if (string.IsNullOrEmpty(line)) + addDependencies = false; + } + return dependencyList; + } + private static Dependency GetDependencyMappings(PythonPackage mainComp, List dependencyList, List AllComps) { List subDependencies = new(); diff --git a/src/LCT.SW360PackageCreator/URLHelper.cs b/src/LCT.SW360PackageCreator/URLHelper.cs index 26141a4f..42719aba 100644 --- a/src/LCT.SW360PackageCreator/URLHelper.cs +++ b/src/LCT.SW360PackageCreator/URLHelper.cs @@ -460,7 +460,7 @@ public async Task GetSourceUrlForPythonPackage(string componentName, str private async Task GetResponseFromPyPiOrg(string componentName, string componenVersion) { string URL; - string result = ""; + const string result = ""; try { URL = $"{CommonAppSettings.PyPiURL}{componentName}" + @@ -480,7 +480,7 @@ private async Task GetResponseFromPyPiOrg(string componentName, string c return result; } - private string GetSourceURLFromPyPiResponse(string response) + private static string GetSourceURLFromPyPiResponse(string response) { string SourceURL = ""; From f6b22c80ce335362c25775c03b65c3259bc8dac1 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 28 Sep 2023 09:56:37 +0530 Subject: [PATCH 18/18] PR Comments --- .../NugetDevDependencyParser.cs | 2 +- src/LCT.PackageIdentifier/NugetProcessor.cs | 2 +- .../appSettingsSW360IntegrationTest.json | 12 ++++++------ src/UnitTestUtilities/appsettingsUnitTest.json | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs index 4ba9c63a..386ffb68 100644 --- a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs +++ b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs @@ -44,7 +44,7 @@ public static NugetDevDependencyParser Instance } } - public static List Parse(string configFile) + public List Parse(string configFile) { List containerList = new(); diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index f4ba45ae..3a800b48 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -633,7 +633,7 @@ private static string ReferenceTagDetailsForPackageReference(XmlNode childNode, private static List ParseAssetFile(string configFile) { NugetDevDependencyParser nugetDevDependencyParser = NugetDevDependencyParser.Instance; - List containers = NugetDevDependencyParser.Parse(configFile); + List containers = nugetDevDependencyParser.Parse(configFile); return ConvertContainerAsNugetPackage(containers, configFile); } diff --git a/src/TestUtilities/appSettingsSW360IntegrationTest.json b/src/TestUtilities/appSettingsSW360IntegrationTest.json index 3ad58a06..5ea55547 100644 --- a/src/TestUtilities/appSettingsSW360IntegrationTest.json +++ b/src/TestUtilities/appSettingsSW360IntegrationTest.json @@ -1,15 +1,15 @@ { - "FossologyURL": "http://md2pdvnc.ad001.siemens.net:8096", + "FossologyURL": "", "RemoveDevDependency": true, "SW360AuthTokenType": "Token", "SW360ProjectID": "0d0e23f6bccb4072be91b5a3462414ad", "SW360ProjectName": "Test", - "SW360URL": "http://md2pdvnc.ad001.siemens.net:8095", + "SW360URL": "", "JfrogNugetDestRepoName": "nuget-test", "JfrogNpmDestRepoName": "npm-test", "JfrogMavenDestRepoName": "maven-test", - "ArtifactoryUploadApiKey": "AKCp8nyNrX7MxdivMy7ka2CwdLfQERMcbJ4Y6icHoQJQA6ZMjYJk4nLTrWs5TCeWXmzz8ya94", - "ArtifactoryUploadUser": "karthika.g@siemens.com", - "JfrogApi": "https://siemens.jfrog.io/artifactory", - "SW360AuthTokenValue": "mlhPtOTmz7cAG7deyyDh" + "ArtifactoryUploadUser": "", + "ArtifactoryUploadApiKey": "", + "JfrogApi": "", + "SW360AuthTokenValue": "" } diff --git a/src/UnitTestUtilities/appsettingsUnitTest.json b/src/UnitTestUtilities/appsettingsUnitTest.json index df660243..4922a603 100644 --- a/src/UnitTestUtilities/appsettingsUnitTest.json +++ b/src/UnitTestUtilities/appsettingsUnitTest.json @@ -1,5 +1,5 @@ { "SW360URL": "http://localhost:8090", "FossologyURL": "http://localhost:8091", - "JFrogURL": "https://siemens.jfrog.io/artifactory" + "JFrogURL": "" } \ No newline at end of file