Move in-vertical manifest merging to new arcade package

eng/Versions.props

@@ -272,7 +272,6 @@
     <MicrosoftDotNetWpfProjectTemplatesPackageVersion>10.0.0-preview.4.25210.1</MicrosoftDotNetWpfProjectTemplatesPackageVersion>
   </PropertyGroup>
   <PropertyGroup Label="Infrastructure and test only dependencies">
-    <VersionToolsVersion>2.2.0-beta.19072.10</VersionToolsVersion>
     <MicrosoftDotNetScenarioTestsSdkTemplateTestsVersion>10.0.0-preview.24602.1</MicrosoftDotNetScenarioTestsSdkTemplateTestsVersion>
   </PropertyGroup>
   <PropertyGroup Label="Manually updated">

src/SourceBuild/content/Directory.Packages.props

@@ -9,7 +9,8 @@
 
   <ItemGroup>
     <!-- Arcade dependencies -->
-    <PackageVersion Include="Microsoft.DotNet.VersionTools" Version="$(MicrosoftDotNetVersionToolsVersion)" />
+    <PackageVersion Include="Microsoft.Arcade.Common" Version="$(MicrosoftArcadeCommonVersion)" />
+    <PackageVersion Include="Microsoft.DotNet.Build.Manifest" Version="$(MicrosoftDotNetBuildManifestVersion)" />
     <!-- Command-line-api dependencies -->
     <PackageVersion Include="System.CommandLine" Version="$(SystemCommandLineVersion)" />
     <!-- MSBuild dependencies -->

src/SourceBuild/content/eng/Version.Details.xml

@@ -6,7 +6,11 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>3fddad170a95109a19a1fee78a83a87cd2e2bb79</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.VersionTools" Version="10.0.0-beta.25110.3">
+    <Dependency Name="Microsoft.DotNet.Build.Manifest" Version="10.0.0-beta.25202.1">
+      <Uri>https://github.com/dotnet/arcade</Uri>
+      <Sha>3fddad170a95109a19a1fee78a83a87cd2e2bb79</Sha>
+    </Dependency>
+    <Dependency Name="Microsoft.Arcade.Common" Version="10.0.0-beta.25202.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>3fddad170a95109a19a1fee78a83a87cd2e2bb79</Sha>
     </Dependency>

src/SourceBuild/content/eng/Versions.props

@@ -26,7 +26,8 @@
     <PrivateSourceBuiltSdkVersion>10.0.100-preview.4.25203.1</PrivateSourceBuiltSdkVersion>
     <PrivateSourceBuiltArtifactsVersion>10.0.100-preview.4.25203.1</PrivateSourceBuiltArtifactsVersion>
     <!-- arcade dependencies -->
-    <MicrosoftDotNetVersionToolsVersion>10.0.0-beta.25110.3</MicrosoftDotNetVersionToolsVersion>
+    <MicrosoftArcadeCommonVersion>10.0.0-beta.25202.1</MicrosoftArcadeCommonVersion>
+    <MicrosoftDotNetBuildManifestVersion>10.0.0-beta.25202.1</MicrosoftDotNetBuildManifestVersion>
     <!-- command-line-api dependencies -->
     <SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
     <!-- msbuild dependencies -->

src/SourceBuild/content/eng/merge-asset-manifests.proj

@@ -17,11 +17,9 @@
 
     <Error Text="Couldn't find any repository asset manifest file. Make sure to build the repositories before invoking this target." Condition="'@(RepoAssetManifest)' == ''" />
 
-    <!-- It's OK for the VmrBuildNumber to be empty -->
     <Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests 
       AssetManifest="@(RepoAssetManifest)" 
       MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
-      VmrBuildNumber="$(BUILD_BUILDNUMBER)"
       VerticalName="$(VerticalName)" />
 
     <!-- Mark the artifact so that it gets binplaced. -->

src/SourceBuild/content/eng/tools/BuildComparer/BuildComparer.csproj

@@ -7,7 +7,8 @@
 	</PropertyGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Microsoft.DotNet.VersionTools" />
+		<PackageReference Include="Microsoft.DotNet.Build.Manifest" />
+		<PackageReference Include="Microsoft.Arcade.Common" />
 		<PackageReference Include="System.CommandLine" />
 	</ItemGroup>
 </Project>

src/SourceBuild/content/eng/tools/BuildComparer/Program.cs

@@ -1,5 +1,5 @@
-using Microsoft.DotNet.VersionTools.Automation;
-using Microsoft.DotNet.VersionTools.BuildManifest;
+using Microsoft.Arcade.Common;
+using Microsoft.DotNet.Build.Manifest;
 using NuGet.Packaging;
 using System.Collections.Concurrent;
 using System.Collections.Immutable;

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownArtifactsFromAssetManifests.cs

@@ -15,7 +15,7 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks;
 /// <summary>
 /// Get a list of MSBuild Items that represent the packages described in the asset manifests.
 /// </summary>
-public sealed class GetKnownArtifactsFromAssetManifests : Build.Utilities.Task
+public sealed class GetKnownArtifactsFromAssetManifests : Task
 {
     // Common metadata
     private const string IdAttributeName = "Id";

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs

@@ -2,17 +2,17 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Microsoft.Build.Utilities;
+using Microsoft.Arcade.Common;
 using Microsoft.Build.Framework;
+using Microsoft.DotNet.Build.Manifest;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using System;
-using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Xml.Linq;
 
 namespace Microsoft.DotNet.UnifiedBuild.Tasks
 {
-    public class MergeAssetManifests : Task
+    public class MergeAssetManifests : MSBuildTaskBase
     {
         /// <summary>
         /// AssetManifest paths
@@ -26,97 +26,67 @@ public class MergeAssetManifests : Task
         [Required]
         public required string MergedAssetManifestOutputPath { get; init; }
 
-        /// <summary>
-        /// Azure DevOps build number
-        /// </summary>
-        public string VmrBuildNumber { get; set; } = string.Empty;
-
         /// <summary>
         /// Vmr Vertical Name, e.g. "Android_Shortstack_arm". Allowed to be empty for non official builds.
         /// </summary>
         public string VerticalName { get; set; } = string.Empty;
 
-        private static readonly string _buildIdAttribute = "BuildId";
         private static readonly string _verticalNameAttribute = "VerticalName";
-        private static readonly string _azureDevOpsBuildNumberAttribute = "AzureDevOpsBuildNumber";
-        private static readonly string[] _ignoredAttributes = [
-            _buildIdAttribute,
-            _azureDevOpsBuildNumberAttribute,
-            "IsReleaseOnlyPackageVersion",
-            "IsStable"
-            ];
-
-        public override bool Execute()
-        {
-            List<XDocument> assetManifestXmls = AssetManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList();
-
-            VerifyAssetManifests(assetManifestXmls);
-
-            XElement mergedManifestRoot = assetManifestXmls.First().Root
-                ?? throw new ArgumentException("The root element of the asset manifest is null.");
-
-            // Set the BuildId and AzureDevOpsBuildNumber attributes to the value of VmrBuildNumber
-            mergedManifestRoot.SetAttributeValue(_buildIdAttribute, VmrBuildNumber);
-            mergedManifestRoot.SetAttributeValue(_azureDevOpsBuildNumberAttribute, VmrBuildNumber);
-            mergedManifestRoot.SetAttributeValue(_verticalNameAttribute, VerticalName);
 
-            List<XElement> packageElements = new();
-            List<XElement> blobElements = new();
+        private IBuildModelFactory? _buildModelFactory;
+        private IFileSystem? _fileSystem;
 
-            foreach (var assetManifestXml in assetManifestXmls)
-            {
-                // We may encounter assets here with "Vertical", "Internal", or "External" visibility here.
-                // We filter out "Vertical" visibility assets here, as they are not needed in the merged manifest.
-                // We leave in "Internal" assets so they can be used in later build passes.
-                // We leave in "External" assets as we will eventually ship them.
-                packageElements.AddRange(assetManifestXml.Descendants("Package").Where(package => package.Attribute("Visibility")?.Value != "Vertical"));
-                blobElements.AddRange(assetManifestXml.Descendants("Blob").Where(blob => blob.Attribute("Visibility")?.Value != "Vertical"));
-            }
-
-            packageElements = packageElements.OrderBy(packageElement => packageElement.Attribute("Id")?.Value).ToList();
-            blobElements = blobElements.OrderBy(blobElement => blobElement.Attribute("Id")?.Value).ToList();
-
-            XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements));
-
-            FileInfo outputFileInfo = new(MergedAssetManifestOutputPath);
-            outputFileInfo.Directory!.Create();
-            File.WriteAllText(outputFileInfo.FullName, verticalManifest.ToString());
+        public override void ConfigureServices(IServiceCollection collection)
+        {
+            collection.TryAddSingleton<IBuildModelFactory, BuildModelFactory>();
+            collection.TryAddSingleton<IBlobArtifactModelFactory, BlobArtifactModelFactory>();
+            collection.TryAddSingleton<IPdbArtifactModelFactory, PdbArtifactModelFactory>();
+            collection.TryAddSingleton<IPackageArtifactModelFactory, PackageArtifactModelFactory>();
+            collection.TryAddSingleton<INupkgInfoFactory, NupkgInfoFactory>();
+            collection.TryAddSingleton<IPackageArchiveReaderFactory, PackageArchiveReaderFactory>();
+            collection.TryAddSingleton<IFileSystem, FileSystem>();
+            collection.TryAddSingleton(Log);
+        }
 
-            Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}");
+        public bool ExecuteTask(IBuildModelFactory buildModelFactory,
+                                IFileSystem fileSystem)
+        {
+            _buildModelFactory = buildModelFactory;
+            _fileSystem = fileSystem;
 
+            MergeManifests();
             return !Log.HasLoggedErrors;
         }
 
-        private static void VerifyAssetManifests(IReadOnlyList<XDocument> assetManifestXmls)
+        public void MergeManifests()
         {
-            if (assetManifestXmls.Count == 0)
+            if (_buildModelFactory == null)
             {
-                throw new ArgumentException("No asset manifests were provided.");
+                throw new InvalidOperationException("BuildModelFactory is not initialized.");
             }
 
-            HashSet<string> rootAttributes = assetManifestXmls
-                .First()
-                .Root?
-                .Attributes()
-                .Select(attribute => attribute.ToString())
-                .ToHashSet()
-                ?? throw new ArgumentException("The root element of the asset manifest is null.");
-
-            if (assetManifestXmls.Skip(1).Any(manifest => manifest.Root?.Attributes().Count() != rootAttributes.Count))
+            if (_fileSystem == null)
             {
-                throw new ArgumentException("The asset manifests do not have the same number of root attributes.");
+                throw new InvalidOperationException("FileSystem is not initialized.");
             }
 
-            if (assetManifestXmls.Skip(1).Any(assetManifestXml =>
-                    !assetManifestXml.Root?.Attributes().Select(attribute => attribute.ToString())
-                        .All(attribute =>
-                            // Ignore BuildId and AzureDevOpsBuildNumber attributes, they're different for different repos,
-                            // TODO this should be fixed with https://github.com/dotnet/source-build/issues/3934
-                            _ignoredAttributes.Any(ignoredAttribute => attribute.StartsWith(ignoredAttribute)) || rootAttributes.Contains(attribute))
-                        ?? false))
-            {
-                throw new ArgumentException("The asset manifests do not have the same root attributes.");
-            }
+            var assetManifestModels = AssetManifest.Select(xmlPath => _buildModelFactory.ManifestFileToModel(xmlPath.ItemSpec))
+                .ToList();
+
+            // We may encounter assets here with "Vertical", "Internal", or "External" visibility here.
+            // We filter out "Vertical" visibility assets here, as they are not needed in the merged manifest.
+            // We leave in "Internal" assets so they can be used in later build passes.
+            // We leave in "External" assets as we will eventually ship them.
+
+            var mergedManifest = _buildModelFactory.CreateMergedModel(assetManifestModels, ArtifactVisibility.Internal | ArtifactVisibility.External);
+
+            // Add a vertical name in the merged model's build identity
+            mergedManifest.Identity.Attributes.Add(_verticalNameAttribute, VerticalName);
+
+            _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(MergedAssetManifestOutputPath)!);
+            _fileSystem.WriteToFile(MergedAssetManifestOutputPath, mergedManifest.ToXml().ToString());
+
+            Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}");
         }
     }
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Microsoft.DotNet.UnifiedBuild.Tasks.csproj

@@ -1,7 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-	  <TargetFramework>$(NetCurrent)</TargetFramework>
+	<TargetFramework>$(NetCurrent)</TargetFramework>
+	<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
   </PropertyGroup>
 
   <ItemGroup>
@@ -13,4 +14,10 @@
     <PackageReference Include="Newtonsoft.Json" IncludeAssets="compile" />
   </ItemGroup>
 
+	<!-- These are actual dependencies -->
+  <ItemGroup>
+	<PackageReference Include="Microsoft.DotNet.Build.Manifest" />
+	<PackageReference Include="Microsoft.Arcade.Common" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -140,6 +140,8 @@
     <!-- Pass the repository URL in globally so that we redirect sourcelink package information to the VMR repo. -->
     <DotNetRepositoryUrl>https://github.com/dotnet/dotnet</DotNetRepositoryUrl>
     <BuildArgs>$(BuildArgs) /p:RepositoryUrl=$(DotNetRepositoryUrl)</BuildArgs>
+    <!-- Force v4 publishing across all repositories. -->
+    <BuildArgs>$(BuildArgs) /p:PublishingVersion=4</BuildArgs>
   </PropertyGroup>
   <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <BuildArgs>$(BuildArgs) /p:SourceBuiltSymbolsDir=$(IntermediateSymbolsRepoDir)</BuildArgs>

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -666,6 +666,7 @@
                     Exists('$(RepoArtifactsDir)')">
     <ItemGroup>
       <LogFilesToMove Include="$(RepoArtifactsDir)**/*.log" />
+      <LogFilesToMove Include="$(RepoArtifactsDir)**/AssetManifest/*.xml" />
       <LogFilesToMove Include="$(RepoArtifactsDir)**/*.binlog" />
       <PrebuiltReportsToMove Include="$(RepoArtifactsDir)sb/prebuilt-report/*" />
     </ItemGroup>

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tasks.Tests/MergeAssetManifestsTests.cs

@@ -0,0 +1,69 @@
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.UnifiedBuild.Tasks;
+using Microsoft.DotNet.UnifiedBuild.Tasks.ManifestAssets;
+using NuGet.ContentModel;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml.Linq;
+using Xunit;
+using Xunit.Abstractions;
+using static System.Net.Mime.MediaTypeNames;
+
+namespace Microsoft.DotNet.Tests
+{
+    /// <summary>
+    /// Tests for the MergeAssetManifests task. This task is used to join the repo builds in a single vertical
+    /// </summary>
+    [Trait("Category", "MergeAssetManifests")]
+    public class MergeAssetManifestsTests
+    {
+        private const string manifestsBasePath = "MergeAssetManifestsTests";
+
+        [Theory]
+        [InlineData("manifests1")]
+        [InlineData("manifests2")]
+        [InlineData("manifests3")]
+        public static void MergeManifestCheck(string manifestSet)
+        {
+            // Load all manifests in the manifests2 directory
+            var manifestPaths = Directory.EnumerateFiles(AssetsLoader.GetAssetFullPath(Path.Combine(manifestsBasePath, manifestSet)), "*.xml")
+                .Where(p => Path.GetFileName(p) != "expected.xml")
+                .Select(p => Path.Combine(manifestsBasePath, manifestSet, p))
+                .ToList();
+            string expectedPath = AssetsLoader.GetAssetFullPath(Path.Combine(manifestsBasePath, manifestSet, "expected.xml"));
+
+            // Create a temporary file to write the merged manifest
+
+            string tempFile = Path.Combine(Path.GetTempFileName());
+
+            try
+            {
+                var task = new MergeAssetManifests
+                {
+                    AssetManifest = manifestPaths.Select(p => new TaskItem(AssetsLoader.GetAssetFullPath(p))).ToArray(),
+                    VerticalName = "Windows_x64",
+                    MergedAssetManifestOutputPath = tempFile,
+                    BuildEngine = new MockBuildEngine()
+                };
+
+                task.Execute();
+
+                Assert.False(task.Log.HasLoggedErrors);
+
+                // Load the merged manifest xml and compare to the expected output.
+                var mergedManifest = XDocument.Load(tempFile);
+                var expectedManifest = XDocument.Load(AssetsLoader.GetAssetFullPath(expectedPath));
+                Assert.Equal(expectedManifest.ToString(), mergedManifest.ToString());
+            }
+            finally
+            {
+                // Clean up the temporary file
+                if (File.Exists(tempFile))
+                {
+                    File.Delete(tempFile);
+                }
+            }
+        }
+    }
+}

src/SourceBuild/content/test/Microsoft.DotNet.Tests/Microsoft.DotNet.Tests.csproj → src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tasks.Tests/Microsoft.DotNet.UnifiedBuild.Tasks.Tests.csproj

@@ -19,11 +19,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="assets\**" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="assets\**" CopyToOutputDirectory="Always" />
   </ItemGroup>
 
-  <Target Name="SetRuntimeConfigOptions"
-          BeforeTargets="_GenerateRuntimeConfigurationFilesInputCache">
+  <Target Name="SetRuntimeConfigOptions" BeforeTargets="_GenerateRuntimeConfigurationFilesInputCache">
     <ItemGroup>
       <!-- General configs -->
       <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).LogsDirectory">