Skip to content

Commit 9e59acb

Browse files
jkoritzinskyjkoritzinsky
authored
Unify dotnet/runtime on using Arcade -publish (#108990)
* Unify dotnet/runtime on using Arcade -publish Fixes dotnet/source-build#4239 Replaces #100004 Co-authored By: Viktor Hofer <[email protected]> * Use DotNetBuildOrchestrator to avoid executing in a repo source-build context. * Actually, we want this logic to run only when we're running in a non-VMR-like context, even repo build. * Restore, but only restore the toolset in the publish job * Fix Exists check and do the "restore toolset only" check one degree away from the Arcade SDK. * Set the "publish using pipelines" global property when publishing * Don't produce the product version files on repo source build.
1 parent e11b867 commit 9e59acb

13 files changed

+207
-461
lines changed

‎Build.proj

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<Project Sdk="Microsoft.Build.Traversal">
22

3-
<ItemGroup>
3+
<ItemGroup Condition="'$(RestoreToolsetOnly)' != 'true'">
44
<!-- Subsets are already imported by Directory.Build.props. -->
55
<ProjectReference Include="@(ProjectToBuild)" />
66
<!-- Only include tasks.proj during restore and build incrementally via a target. -->

‎Directory.Build.targets

-26
Original file line numberDiff line numberDiff line change
@@ -194,30 +194,4 @@
194194
<Analyzer Remove="@(_targetingPackExcludedAnalyzerReferenceWithProjectName->Metadata('OriginalIdentity'))" />
195195
</ItemGroup>
196196
</Target>
197-
198-
<!--
199-
Arcade SDK versioning is defined by static properties in a targets file: work around this by
200-
moving properties based on versioning into a target.
201-
-->
202-
<Target Name="GetProductVersions">
203-
<PropertyGroup>
204-
<IncludePreReleaseLabelInPackageVersion Condition="'$(DotNetFinalVersionKind)' != 'release'">true</IncludePreReleaseLabelInPackageVersion>
205-
<IncludePreReleaseLabelInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludePreReleaseLabelInPackageVersion>
206-
<IncludePreReleaseLabelInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludePreReleaseLabelInPackageVersion>
207-
208-
<IncludeBuildNumberInPackageVersion Condition="'$(StabilizePackageVersion)' != 'true'">true</IncludeBuildNumberInPackageVersion>
209-
<IncludeBuildNumberInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludeBuildNumberInPackageVersion>
210-
<IncludeBuildNumberInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludeBuildNumberInPackageVersion>
211-
212-
<ProductVersionSuffix Condition="'$(IncludePreReleaseLabelInPackageVersion)' == 'true'">-$(VersionSuffix)</ProductVersionSuffix>
213-
<ProductBandVersion Condition="'$(ProductBandVersion)' == ''">$(MajorVersion).$(MinorVersion)</ProductBandVersion>
214-
<ProductionVersion Condition="'$(ProductionVersion)' == ''">$(ProductBandVersion).$(PatchVersion)</ProductionVersion>
215-
<ProductVersion>$(ProductionVersion)$(ProductVersionSuffix)</ProductVersion>
216-
217-
<SharedFrameworkNugetVersion>$(ProductVersion)</SharedFrameworkNugetVersion>
218-
<NuGetVersion>$(SharedFrameworkNugetVersion)</NuGetVersion>
219-
<InstallersRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</InstallersRelativePath>
220-
</PropertyGroup>
221-
</Target>
222-
223197
</Project>

‎eng/Publishing.props

+189-22
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,206 @@
1-
<Project>
1+
<Project InitialTargets="ValidateDownloadedAssets">
22

33
<PropertyGroup>
44
<ProducesDotNetReleaseShippingAssets>true</ProducesDotNetReleaseShippingAssets>
5+
<!-- This avoids creating VS.*.symbols.nupkg packages that are identical to the original package. -->
6+
<AutoGenerateSymbolPackages>false</AutoGenerateSymbolPackages>
7+
<!-- Set PlatformName to TargetArchitecture to create unique build manifest files. -->
8+
<PlatformName Condition="'$(TargetArchitecture)' != ''">$(TargetArchitecture)</PlatformName>
59
</PropertyGroup>
610

7-
<!-- Include installer archives and packages which aren't globbed by default.
8-
Don't include Symbols archive as it is already included in Arcade's Publish.proj, with correct blob path. -->
9-
<Target Name="PublishRuntimeInstallers"
10-
BeforeTargets="BeforePublish"
11-
Condition="'$(DotNetBuildRepo)' == 'true'">
12-
<!-- Retrieve runtime's runtime pack product version.
11+
<!--
12+
Look through the downloaded artifacts to figure out how to ship them. Creates item groups for
13+
other types of publishing to use.
14+
15+
DownloadDirectory is expected to have some directory inside, which then contains a dir for each
16+
build job name. For example, this nupkg would be found:
17+
18+
$(DownloadDirectory)IntermediateArtifacts\windows_x64\Shipping\a.nupkg
19+
-->
20+
<ItemGroup Condition="'$(DotNetBuildRepo)' != 'true'">
21+
<DownloadedArtifactFile Include="$(DownloadDirectory)**" />
22+
<DownloadedSymbolNupkgFile Include="$(DownloadDirectory)**\*.symbols.nupkg" />
23+
<DownloadedWixPdbFile Include="$(DownloadDirectory)**\*.wixpdb" />
24+
<DownloadedWixpackFile Include="$(DownloadDirectory)**\*.wixpack.zip" Condition="'$(PostBuildSign)' != 'true'" />
25+
<DownloadedWorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*" />
26+
<DownloadedNupkgFile Include="$(DownloadDirectory)**\*.nupkg" Exclude="@(DownloadedSymbolNupkgFile)" />
27+
28+
<!-- Exclude symbol packages from have a NuGet signature. These are never pushed to NuGet.org or
29+
other feeds (in fact, that have identical identity to their non-symbol variant) -->
30+
<DownloadedSymbolPackagesWithoutPaths Include="@(DownloadedSymbolNupkgFile->'%(Filename)%(Extension)')" />
31+
<FileSignInfo Include="@(DownloadedSymbolPackagesWithoutPaths->Distinct())" CertificateName="None" />
32+
33+
<!-- Add files that are not affected by filtering and create checksum for them. -->
34+
<UploadToBlobStorageFile
35+
Include="@(DownloadedArtifactFile)"
36+
Exclude="@(DownloadedSymbolNupkgFile);
37+
@(DownloadedNupkgFile);
38+
@(DownloadedWixPdbFile);
39+
@(DownloadedWorkloadsVSInsertionFile);
40+
@(DownloadedWixpackFile)" />
41+
42+
<!--
43+
Filter out the RID-specific (Runtime) nupkgs and RID-agnostic nupkgs. RID-specific packages
44+
are published from every job. RID-agnostic nupkgs are built with the same ID/version by
45+
every job, so one specific job's outputs must be picked to sign and publish.
46+
-->
47+
48+
<!-- RID-specific framework packs. -->
49+
<RuntimeNupkgFile
50+
Include="
51+
$(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
52+
$(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg;
53+
$(DownloadDirectory)**\Microsoft.*.App.Crossgen2.*.nupkg"
54+
Exclude="@(DownloadedSymbolNupkgFile)" />
55+
56+
<!-- VS insertion packages, carrying RID-specific installers. -->
57+
<RuntimeNupkgFile
58+
Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
59+
Exclude="@(DownloadedSymbolNupkgFile)" />
60+
61+
<!--
62+
Workloads VS insertion artifacts produced by src/workloads/workloads.csproj. Only grab
63+
the zip artifacts as they're grouped by SDK feature band which correlates with specific VS versions.
64+
-->
65+
<WorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*.zip" />
66+
67+
<!--
68+
Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
69+
because Libraries produces some "runtime.native.Foo" packages with
70+
"runtime.<rid>.runtime.native.Foo" identity packages.
71+
-->
72+
<RuntimeNupkgFile
73+
Include="$(DownloadDirectory)**\runtime.*.nupkg"
74+
Exclude="
75+
$(DownloadDirectory)**\runtime.native.*.nupkg;
76+
@(DownloadedSymbolNupkgFile)" />
77+
78+
<!--
79+
Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
80+
the AllConfigurations packages from the Libraries build.
81+
-->
82+
<RidAgnosticNupkgToPublishFile
83+
Include="
84+
$(DownloadDirectory)**\Microsoft.NET.Workload.Mono.Toolchain.*Manifest-*.nupkg;
85+
$(DownloadDirectory)**\Microsoft.NET.Sdk.WebAssembly.Pack.*.nupkg;
86+
$(DownloadDirectory)*\$(PublishRidAgnosticPackagesFromPlatform)\**\*.nupkg;
87+
$(DownloadDirectory)*\*AllConfigurations\**\*.nupkg"
88+
Exclude="@(RuntimeNupkgFile);@(DownloadedSymbolNupkgFile)" />
89+
90+
<TransportPackagesToPublishFile
91+
Include="$(DownloadDirectory)**\*Transport*.nupkg"
92+
Exclude="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(DownloadedSymbolNupkgFile)" />
93+
94+
<NupkgToPublishFile Include="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(TransportPackagesToPublishFile)" />
95+
96+
<!--
97+
Assuming all symbol packages ship and can be found by turning .nupkg => .symbols.nupkg, find
98+
them. Don't check for missing symbol packages here: some nupkgs don't have them for valid
99+
reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
100+
will check for symbol completeness with file-by-file granularity rather than looking for
101+
missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.
102+
103+
Handles several conventions:
104+
* NonShipping packages have symbol nupkgs that are Shipping.
105+
* Shipping packages have symbol packages in a "symbols" subdirectory.
106+
-->
107+
<PotentialSymbolNupkgToPublishFile
108+
Include="
109+
@(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
110+
@(NupkgToPublishFile->Replace('\NonShipping\', '\NonShipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'));
111+
@(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />
112+
113+
<SymbolNupkgToPublishFile Include="@(PotentialSymbolNupkgToPublishFile->Distinct()->Exists())" />
114+
115+
<!-- Packages -->
116+
<Artifact Include="@(NupkgToPublishFile)"
117+
IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
118+
PublishFlatContainer="false" />
119+
<Artifact Include="@(SymbolNupkgToPublishFile)" PublishFlatContainer="false" />
120+
121+
<!-- Blob storage -->
122+
<Artifact Include="@(UploadToBlobStorageFile)"
123+
Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)"
124+
IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
125+
ChecksumPath="%(FullPath).sha512" />
126+
<Artifact Include="@(WorkloadsVSInsertionFile)"
127+
SubBlobFolder="workloads/"
128+
IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))" />
129+
</ItemGroup>
130+
131+
<Target Name="GetNonStableProductVersion">
132+
<!-- Retrieve the non-stable runtime pack product version.
13133
Don't stabilize the package version in order to retrieve the VersionSuffix. -->
14134
<MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
15135
Targets="ReturnProductVersion"
16136
Properties="IsShipping=false;
17137
Crossgen2SdkOverridePropsPath=;
18138
Crossgen2SdkOverrideTargetsPath=">
19-
<Output TaskParameter="TargetOutputs" PropertyName="RuntimeRuntimePackProductVersion" />
139+
<Output TaskParameter="TargetOutputs" PropertyName="NonStableProductVersion" />
140+
</MSBuild>
141+
</Target>
142+
143+
<!--
144+
Only generate the productVersion.txt and runtime-productVersion.txt files when we're building
145+
either the full VMR repo or not building in the VMR infrastructure.
146+
This ensures that we don't produce these files in the "Repo source build" builds,
147+
but we do produce them in both the VMR and the runtime official build.
148+
-->
149+
<PropertyGroup>
150+
<_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' == 'true' and '$(DotNetBuildOrchestrator)' == 'true'">true</_ShouldGenerateProductVersionFiles>
151+
<_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' != 'true' and '$(DotNetBuildOrchestrator)' != 'true'">true</_ShouldGenerateProductVersionFiles>
152+
</PropertyGroup>
153+
154+
<Target Name="GenerateProductVersionFiles"
155+
DependsOnTargets="GetNonStableProductVersion"
156+
BeforeTargets="PublishToAzureDevOpsArtifacts"
157+
Condition="'$(_ShouldGenerateProductVersionFiles)' == 'true'">
158+
<!-- Retrieve the runtime pack product version. -->
159+
<MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
160+
Targets="ReturnProductVersion"
161+
Properties="Crossgen2SdkOverridePropsPath=;
162+
Crossgen2SdkOverrideTargetsPath=">
163+
<Output TaskParameter="TargetOutputs" PropertyName="ProductVersionForVersionsFile" />
20164
</MSBuild>
21165

22166
<ItemGroup>
23-
<InstallerToPublish Include="$(ArtifactsPackagesDir)**\*.tar.gz;
24-
$(ArtifactsPackagesDir)**\*.zip;
25-
$(ArtifactsPackagesDir)**\*.deb;
26-
$(ArtifactsPackagesDir)**\*.rpm;
27-
$(ArtifactsPackagesDir)**\*.pkg;
28-
$(ArtifactsPackagesDir)**\*.exe;
29-
$(ArtifactsPackagesDir)**\*.msi"
30-
Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" />
31-
32-
<ItemsToPushToBlobFeed Include="@(InstallerToPublish)"
33-
IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"
34-
PublishFlatContainer="true"
35-
RelativeBlobPath="Runtime/$(RuntimeRuntimePackProductVersion)/%(Filename)%(Extension)" />
167+
<ProductVersionFile Include="$(ArtifactsShippingPackagesDir)productVersion.txt" />
168+
<ProductVersionFile Include="$(ArtifactsShippingPackagesDir)runtime-productVersion.txt" />
36169
</ItemGroup>
170+
171+
<!-- Generate productVersion.txt file containing the product version. -->
172+
<WriteLinesToFile File="%(ProductVersionFile.Identity)"
173+
Lines="$(ProductVersionForVersionsFile)"
174+
Overwrite="true"
175+
Encoding="ASCII" />
176+
177+
<ItemGroup>
178+
<Artifact Include="@(ProductVersionFile)"
179+
RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Filename)%(Extension)" />
180+
</ItemGroup>
181+
</Target>
182+
183+
<Target Name="AddRelativeBlobPathToInstallerArtifacts"
184+
DependsOnTargets="GetNonStableProductVersion"
185+
BeforeTargets="PublishToAzureDevOpsArtifacts"
186+
AfterTargets="GenerateChecksumsFromArtifacts">
187+
<ItemGroup>
188+
<Artifact Condition="'%(Artifact.PublishFlatContainer)' == 'true' and '%(Artifact.RelativeBlobPath)' == ''"
189+
RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Artifact.SubBlobFolder)%(Filename)%(Extension)" />
190+
</ItemGroup>
191+
</Target>
192+
193+
<Target Name="ValidateDownloadedAssets" Condition="'$(DotNetBuildRepo)' != 'true'">
194+
<Error Text="DownloadDirectory is not set." Condition="'$(DownloadDirectory)' == ''" />
195+
<Error Condition="'@(SymbolNupkgToPublishFile)' == ''" Text="No symbol packages found." />
196+
197+
<!--
198+
Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
199+
of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
200+
however the patterns should be fairly specific: this is likely a build infra mistake that
201+
should be corrected.
202+
-->
203+
<Error Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)" Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
37204
</Target>
38205

39-
</Project>
206+
</Project>

‎eng/Subsets.props

-5
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,6 @@
206206
<SubsetName Include="Packs.Tests" Description="The framework pack tests." />
207207

208208
<!-- Utility -->
209-
<SubsetName Include="publish" OnDemand="true" Description="Generate asset manifests and prepare to publish to BAR." />
210209
<SubsetName Include="RegenerateDownloadTable" OnDemand="true" Description="Regenerates the nightly build download table" />
211210
<SubsetName Include="RegenerateThirdPartyNotices" OnDemand="true" Description="Regenerates the THIRD-PARTY-NOTICES.TXT file based on other repos' TPN files." />
212211
<SubsetName Include="tasks" OnDemand="true" Description="Build the repo local task projects." />
@@ -560,10 +559,6 @@
560559
<ProjectToBuild Include="@(TestProjectToBuild)" BuildInParallel="true" Test="true" Category="packs" />
561560
</ItemGroup>
562561

563-
<ItemGroup Condition="$(_subset.Contains('+publish+'))">
564-
<ProjectToBuild Include="$(InstallerProjectRoot)prepare-artifacts.proj" Pack="true" Category="publish" />
565-
</ItemGroup>
566-
567562
<!-- Utility -->
568563
<ItemGroup Condition="$(_subset.Contains('+regeneratedownloadtable+'))">
569564
<ProjectToBuild Include="$(RepositoryEngineeringDir)regenerate-download-table.proj" Pack="true" />

‎eng/pipelines/official/jobs/prepare-signed-artifacts.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,14 @@ jobs:
4343

4444
steps:
4545
- script: >-
46-
build.cmd -ci
47-
-subset publish
48-
-configuration Release
46+
build.cmd -restore -sign -publish -ci -configuration Release
47+
/p:RestoreToolsetOnly=true
4948
/p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }}
49+
/p:DownloadDirectory=$(Build.SourcesDirectory)\artifacts\PackageDownload\
5050
/p:OfficialBuildId=$(Build.BuildNumber)
5151
/p:SignType=$(_SignType)
5252
/p:DotNetSignType=$(_SignType)
53+
/p:DotNetPublishUsingPipelines=true
5354
/bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog
5455
displayName: Prepare artifacts and upload to build
5556

‎src/installer/Directory.Build.props

-7
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,4 @@
1414
want to ensure we use the correct packages. -->
1515
<DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
1616
</PropertyGroup>
17-
18-
<!-- Construct the path to the sign project, so that signing can be called during prepare-artifacts -->
19-
<PropertyGroup>
20-
<ArcadeSdkMSBuildProjectDir>$([System.IO.Path]::GetDirectoryName('$(ArcadeSdkBuildTasksAssembly)'))\..\</ArcadeSdkMSBuildProjectDir>
21-
<ArcadeSdkSignProject>$(ArcadeSdkMSBuildProjectDir)Sign.proj</ArcadeSdkSignProject>
22-
</PropertyGroup>
23-
2417
</Project>

‎src/installer/pkg/projects/Directory.Build.targets

+3-15
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
If we are stabilizing set the StableVersion property for the packages.
66
Needs to happen in Directory.Build.targets to allow all the pkgproj's to set Version property first.
77
-->
8-
<StableVersion Condition="'$(IncludePreReleaseLabelInPackageVersion)' != 'true'">$(Version)</StableVersion>
8+
<StableVersion Condition="'$(DotNetFinalVersionKind)' == 'release' and
9+
'$(SuppressFinalPackageVersion)' != 'true' and
10+
'$(IsShipping)' == 'true'">$(Version)</StableVersion>
911

1012
<!--
1113
Explicitly re-set the symbol package output path. The pkgproj files here end up importing the targets from
@@ -16,20 +18,6 @@
1618
<SymbolPackageOutputPath>$(PackageOutputPath)</SymbolPackageOutputPath>
1719
</PropertyGroup>
1820

19-
<Target Name="SetTargetBasedPackageVersion"
20-
BeforeTargets="GenerateNuSpec"
21-
DependsOnTargets="GetProductVersions">
22-
<PropertyGroup>
23-
<Version>$(ProductVersion)</Version>
24-
<!--
25-
PackageVersion is normally calculated using Version during static property evaluation, but
26-
we need some info from GetProductVersions, so it's too late to rely on that. We need to set
27-
both in target evaluation, here.
28-
-->
29-
<PackageVersion>$(ProductVersion)</PackageVersion>
30-
</PropertyGroup>
31-
</Target>
32-
3321
<!--
3422
Remove duplicate files returned by restore. The resolve task performs extra detection to pick up
3523
a PDB file for any file listed in the assets file. This causes duplicates if the assets file

0 commit comments

Comments
 (0)