Skip to content

Commit

Permalink
Merge branch 'conanpackageDownloader' of https://github.com/siemens/c…
Browse files Browse the repository at this point in the history 
…ontinuous-clearing into conanpackageDownloader
  • Loading branch information
“sagar committed Nov 9, 2023
2 parents 872aba4 + e79083a commit dab9949
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 15 deletions.
57 changes: 57 additions & 0 deletions src/AritfactoryUploader.UTest/ArtifactoryUTTestFiles/CyclonedxBom.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,63 @@
}
],
"Evidence": null
},
{
"Type": 0,
"MimeType": null,
"BomRef": "pkg:conan/[email protected]",
"Supplier": null,
"Author": null,
"Publisher": null,
"Group": null,
"Name": "protobuf",
"Version": "3.21.9",
"Description": "",
"Scope": null,
"Hashes": null,
"Licenses": null,
"Copyright": null,
"Cpe": null,
"Purl": "pkg:conan/[email protected]",
"Swid": null,
"Modified": null,
"Pedigree": null,
"Components": null,
"Properties": [
{
"Name": "internal:siemens:clearing:development",
"Value": "false"
},
{
"Name": "internal:siemens:clearing:identifier-type",
"Value": "Discovered"
},
{
"Name": "internal:siemens:clearing:is-internal",
"Value": "false"
},
{
"Name": "internal:siemens:clearing:repo-url",
"Value": "energy-dev-conan-egll"
},
{
"Name": "internal:siemens:clearing:project-type",
"Value": "CONAN"
},
{
"Name": "internal:siemens:clearing:clearing-state",
"Value": "Approved"
},
{
"Name": "internal:siemens:clearing:sw360:release-url",
"Value": "http://md2pdvnc.ad001.siemens.net:8095/resource/api/releases/e13e0e564b004ef4adabbd01bf0b93ce"
},
{
"Name": "internal:siemens:clearing:fossology:url",
"Value": null
}
],
"Evidence": null
}
],
"Compositions": null
Expand Down
2 changes: 1 addition & 1 deletion src/AritfactoryUploader.UTest/PackageUploadHelperTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public void GetComponentListFromComparisonBOM_GivenComparisonBOM_ReturnsComponen
//Act
Bom componentList = PackageUploadHelper.GetComponentListFromComparisonBOM(comparisonBOMPath);
// Assert
Assert.That(11, Is.EqualTo(componentList.Components.Count), "Checks for no of components");
Assert.That(12, Is.EqualTo(componentList.Components.Count), "Checks for no of components");
}

[Test]
Expand Down
2 changes: 1 addition & 1 deletion src/AritfactoryUploader.UTest/PackageUploaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public async Task UploadPackageToArtifactory_GivenAppsettings()
await PackageUploader.UploadPackageToArtifactory(CommonAppSettings);

// Assert
Assert.That(11, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesToBeUploaded), "Checks for no of components");
Assert.That(12, Is.EqualTo(PackageUploader.uploaderKpiData.PackagesToBeUploaded), "Checks for no of components");
}
}
}
80 changes: 70 additions & 10 deletions src/ArtifactoryUploader/PackageUploadHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public async static Task<List<ComponentsToArtifactory>> GetComponentsToBeUploade
{
if (item.Properties.Exists(p => p.Name == Dataconstant.Cdx_ClearingState && p.Value.ToUpperInvariant() == "APPROVED"))
{
AqlResult aqlResult = await GetSrcRepoDetailsForPyPiPackages(item, appSettings);
AqlResult aqlResult = await GetSrcRepoDetailsForPyPiOrConanPackages(item, appSettings);
ComponentsToArtifactory components = new ComponentsToArtifactory()
{
Name = !string.IsNullOrEmpty(item.Group) ? $"{item.Group}/{item.Name}" : item.Name,
Expand All @@ -85,9 +85,21 @@ public async static Task<List<ComponentsToArtifactory>> GetComponentsToBeUploade
ApiKey = appSettings.ArtifactoryUploadApiKey,
Email = appSettings.ArtifactoryUploadUser,
JfrogApi = appSettings.JFrogApi,
SrcRepoPathWithFullName = aqlResult != null ? aqlResult.Repo + "/" + aqlResult.Path + "/" + aqlResult.Name : string.Empty,
PypiCompName = aqlResult != null ? aqlResult.Name : string.Empty
};

if (aqlResult != null)
{
components.SrcRepoPathWithFullName = aqlResult.Repo + "/" + aqlResult.Path + "/" + aqlResult.Name;
components.Path = GetConanPath(aqlResult.Path, $"{item.Name}/{item.Version}");
components.PypiCompName = aqlResult.Name;
}
else
{
components.SrcRepoPathWithFullName = string.Empty;
components.Path = string.Empty;
components.PypiCompName = string.Empty;
}

components.PackageInfoApiUrl = GetPackageInfoURL(components);
components.CopyPackageApiUrl = GetCopyURL(components);
componentsToBeUploaded.Add(components);
Expand Down Expand Up @@ -126,13 +138,32 @@ private static string GetCopyURL(ComponentsToArtifactory component)
url = $"{component.JfrogApi}{ApiConstant.CopyPackageApi}{component.SrcRepoPathWithFullName}" +
$"?to=/{component.DestRepoName}/{component.PypiCompName}";
}
else if (component.ComponentType == "CONAN")
{
url = $"{component.JfrogApi}{ApiConstant.CopyPackageApi}{component.SrcRepoName}/{component.Path}" +
$"?to=/{component.DestRepoName}/{component.Path}";
}
else
{
// Do nothing
}
return url;
}

private static string GetConanPath(string path, string package)
{
//// Get Path only till PackageName/Version so that everything in folder can be copied
if (path.Contains(package))
{
int index = path.IndexOf(package);
return path.Substring(0, index + package.Length);
}
else
{
return path;
}
}

private static string GetPackageInfoURL(ComponentsToArtifactory component)
{
string url = string.Empty;
Expand All @@ -152,6 +183,10 @@ private static string GetPackageInfoURL(ComponentsToArtifactory component)
{
url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoPathWithFullName}";
}
else if (component.ComponentType == "CONAN")
{
url = $"{component.JfrogApi}{ApiConstant.PackageInfoApi}{component.SrcRepoName}/{component.Path}";
}
else
{
// Do nothing
Expand All @@ -177,16 +212,20 @@ private static string GetDestinationRepo(Component item, CommonAppSettings appSe
{
return appSettings.JfrogPythonDestRepoName;
}
else if (item.Purl.Contains("conan", StringComparison.OrdinalIgnoreCase))
{
return appSettings.JfrogConanDestRepoName;
}
else
{
// Do nothing
}

return string.Empty;
}

private static string GetComponentType(Component item)
{

if (item.Purl.Contains("npm", StringComparison.OrdinalIgnoreCase))
{
return "NPM";
Expand All @@ -203,26 +242,39 @@ private static string GetComponentType(Component item)
{
return "PYTHON";
}
else if (item.Purl.Contains("conan", StringComparison.OrdinalIgnoreCase))
{
return "CONAN";
}
else
{
// Do nothing
}
return string.Empty;
}

private async static Task<AqlResult> GetSrcRepoDetailsForPyPiPackages(Component item, CommonAppSettings appSettings)
private async static Task<AqlResult> GetSrcRepoDetailsForPyPiOrConanPackages(Component item, CommonAppSettings appSettings)
{
if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase) && aqlResultList.Count == 0)
{
// get the component list from Jfrog for given repo
aqlResultList = await GetListOfComponentsFromRepo(appSettings.Python?.JfrogPythonRepoList, jFrogService);
}

if (aqlResultList.Count > 0)
{
return GetArtifactoryRepoName(aqlResultList, item);
if (aqlResultList.Count > 0)
{
return GetArtifactoryRepoName(aqlResultList, item);
}
}
else if (item.Purl.Contains("conan", StringComparison.OrdinalIgnoreCase))
{
var aqlConanResultList = await GetListOfComponentsFromRepo(appSettings.Conan?.JfrogConanRepoList, jFrogService);

if (aqlConanResultList.Count > 0)
{
return GetArtifactoryRepoNameForConan(aqlConanResultList, item);
}
}

return null;
}

Expand Down Expand Up @@ -340,6 +392,14 @@ private static AqlResult GetArtifactoryRepoName(List<AqlResult> aqlResultList, C
return repoName;
}

}
private static AqlResult GetArtifactoryRepoNameForConan(List<AqlResult> aqlResultList, Component component)
{
string jfrogcomponentPath = $"{component.Name}/{component.Version}";

AqlResult repoName = aqlResultList.Find(x => x.Path.Contains(
jfrogcomponentPath, StringComparison.OrdinalIgnoreCase));

return repoName;
}
}
}
2 changes: 1 addition & 1 deletion src/LCT.APICommunications/Model/ComponentsToArtifactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,6 @@ public class ComponentsToArtifactory
public string PackageInfoApiUrl { get; set; }
public string CopyPackageApiUrl { get; set; }
public string PackageExtension { get; set; }

public string Path { get; set; }
}
}
2 changes: 2 additions & 0 deletions src/LCT.Common/CommonAppSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public CommonAppSettings(IFolderAction iFolderAction)
public Config Maven { get; set; }
public Config Debian { get; set; }
public Config Python { get; set; }
public Config Conan { get; set; }
public string CaVersion { get; set; }
public string CycloneDxSBomTemplatePath { get; set; }
public string[] InternalRepoList { get; set; }
Expand All @@ -74,6 +75,7 @@ public CommonAppSettings(IFolderAction iFolderAction)
public string JfrogNugetDestRepoName { get; set; }
public string JfrogMavenDestRepoName { get; set; }
public string JfrogPythonDestRepoName { get; set; }
public string JfrogConanDestRepoName { get; set; }
public string JfrogNugetSrcRepo { get; set; }
public string Mode { get; set; } = string.Empty;

Expand Down
1 change: 1 addition & 0 deletions src/LCT.Common/Model/Config.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public class Config
public string[] JfrogNugetRepoList { get; set; }
public string[] JfrogMavenRepoList { get; set; }
public string[] JfrogPythonRepoList { get; set; }
public string[] JfrogConanRepoList { get; set; }
public string[] DevDependentScopeList { get; set; }

}
Expand Down
3 changes: 2 additions & 1 deletion src/LCT.Common/appSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"JfrogNpmDestRepoName": "<Insert NpmDestRepoName>",
"JfrogMavenDestRepoName": "<Insert MavenDestRepoName>",
"JfrogPythonDestRepoName": "<Insert PythonDestRepoName>",
"JfrogConanDestRepoName": "<Insert ConanDestRepoName>",
"PackageFilePath": "/PathToInputDirectory", //For Docker run set as /mnt/Input
"BomFolderPath": "/PathToOutputDirectory", //For Docker run set as /mnt/Output
"BomFilePath": "/PathToOutputDirectory/<SW360 Project Name>_Bom.cdx.json",
Expand Down Expand Up @@ -70,7 +71,7 @@
"Python": {
"Include": [ "poetry.lock", "*.cdx.json" ],
"Exclude": [],
"JfrogPythonRepoList": [
"JfrogPythonRepoList": [
"<Python Remote Cache Repo Name>", //This is a mirror repo for pypi in JFrog
"<Python Release Repo Name>" //This should be the release pypi in JFrog
],
Expand Down
2 changes: 1 addition & 1 deletion src/LCT.SW360PackageCreator/URLHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ public async Task<string> GetSourceUrlForConanPackage(string componentName, stri
}
if (packageSourcesInfo.SourcesData.TryGetValue(version,out var release))
{
if (release.Url.GetType().Name == "string")
if (release.Url.GetType().Name.ToLowerInvariant() == "string")
{
componentSrcURL = release.Url.ToString();
}
Expand Down

0 comments on commit dab9949

Please sign in to comment.