[build] Sanitize Microsoft.Build* assembly references (#3995)

Context: https://github.com/xamarin/monodroid/pull/1048
Context: #3928

Attempt to clean up and modernize various projects to help break up
the scope of change required to convert them to .NET Standard libs.

Bump the .NET Core version that `build.groovy` and
`azure-pipelines.yaml` install to 3.1.100.  This is to obtain various
NuGet support fixes.

Updates the shared `MSBuildReferences.projitems` file to instead
include NuGet `@(PackageReference)`s to the required
`Microsoft.Build.*.dll` assemblies.  The `Mono.Posix.NETStandard`,
`Xamarin.Build.AsyncTask`, and `Xamarin.LibZipSharp` NuGet references
have also been moved into this file to make it easier to update
versions in one place as needed.  We're only using the compile time
assets from the MSBuild NuGet packages, which ensures that their
content is not copied to the output directory.  An additional `@(None)`
item group has been added to this file to copy the required assemblies
from the system to the output directory so that
`Xamarin.Android.Build.Tests.dll` can run against them.  These changes
are required because the MSBuild assemblies coming from NuGet do not
yet fully support running on Mono.

All projects which reference `MSBuildReferences.projitems` have been
converted from the old `packages.config` to the new
`@(PackageReference)` NuGet reference style.

The Azure Pipeline build job has been updated to completely build
`Xamarin.Android-Tests.sln` with `xabuild`, rather than only building
select projects.  This was done to mirror our Jenkins build and to
ensure that `make all-tests` continues to work moving forward.

.NET Core and `NuGetCommand` tasks have been bumped to bring in a
newer version of NuGet with better `@(PackageReference)` support.

NuGet restore will be skipped if any issues occur when calculating the
dependency graph for the solution or project that it is operating on.
A lot of our sources attempt to import `.props`/`.targets`/`.projitems`
files that are generated by the build, and these should be
conditionally imported so that NuGet restore attempts can succeed
before these files are generated.  This should also help when loading
these projects in VS or VS Mac.

Example NuGet restore error:

	src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets(8,3): error MSB4019: The imported project "bin/BuildRelease/ProfileAssemblies.projitems" was not found. Confirm that the expression in the Import declaration "../../bin/BuildRelease/ProfileAssemblies.projitems" is correct, and that the file exists on disk.
	/var/folders/pp/x88nqyvd0h18ttl996vv2c3w0000gn/T/NuGetScratch/ok90j1ok.3t3.nugetrestore.targets(266,5): warning : Skipping restore for project 'src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj'. The project file may be invalid or missing targets required for restore.

I've also moved the `NuGet.exe` that is copied and used by tests into
its own folder to avoid a crash on macOS which would occur when
`NuGet.exe` is invoked from the same folder as
`Xunit.NetCore.Extensions.dll`:

	mono bin/TestRelease/NuGet.exe

	VTable setup of type Xunit.NetCore.Extensions.SkippedTestCase failed
	System.TypeLoadException: VTable setup of type Xunit.NetCore.Extensions.SkippedTestCase failed
	  at (wrapper managed-to-native) System.RuntimeType.GetPropertiesByName_native(System.RuntimeType,intptr,System.Reflection.BindingFlags,System.RuntimeType/MemberListType)
	  at System.RuntimeType.GetPropertiesByName (System.String name, System.Reflection.BindingFlags bindingAttr, System.RuntimeType+MemberListType listType, System.RuntimeType reflectedType)
	  at System.RuntimeType.GetPropertyCandidates (System.String name, System.Reflection.BindingFlags bindingAttr, System.Type[] types, System.Boolean allowPrefixLookup)
	  at System.RuntimeType.GetProperties (System.Reflection.BindingFlags bindingAttr)
	  at System.ComponentModel.Composition.AttributedModel.AttributedPartCreationInfo+<GetExportMembers>d__32.MoveNext ()
	  at System.Linq.Enumerable.Any[TSource] (System.Collections.Generic.IEnumerable1[T] source)
      ...

Author: pjcollins

Makefile

@@ -26,7 +26,7 @@ _PREPARE_ARGS =
 
 BOOTSTRAP_SOLUTION = Xamarin.Android.BootstrapTasks.sln
 BOOTSTRAP_BUILD_LOG = bin/Build$(CONFIGURATION)/bootstrap-build.binlog
-BOOTSTRAP_MSBUILD_FLAGS = /binaryLogger:"$(BOOTSTRAP_BUILD_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
+BOOTSTRAP_MSBUILD_FLAGS = /t:Restore,Build /binaryLogger:"$(BOOTSTRAP_BUILD_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
 
 all:
 	$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj

@@ -35,12 +35,6 @@
     <Reference Include="System.Net.Http" />
     <Reference Include="System.Xml" />
     <Reference Include="System.Xml.Linq" />
-    <Reference Include="Mono.Posix.NETStandard">
-      <HintPath>..\..\packages\Mono.Posix.NETStandard.1.0.0\lib\net40\Mono.Posix.NETStandard.dll</HintPath>
-    </Reference>
-    <Reference Include="libZipSharp">
-      <HintPath>..\..\packages\Xamarin.LibZipSharp.1.0.6\lib\net45\libZipSharp.dll</HintPath>
-    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -67,6 +61,10 @@
     <Compile Include="Xamarin.Android.Tools.BootstrapTasks\RunInstrumentationTests.cs" />
     <Compile Include="Xamarin.Android.Tools.BootstrapTasks\JdkInfo.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.ApiCompat" Version="5.0.0-beta.19606.1" IncludeAssets="none" />
+    <PackageReference Include="Microsoft.DotNet.GenAPI" Version="5.0.0-beta.19606.1" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\xa-prep-tasks\xa-prep-tasks.csproj">
       <Project>{7CE69551-BD73-4726-ACAA-AAF89C84BAF8}</Project>
@@ -77,10 +75,6 @@
       <Name>Xamarin.Android.Tools.AndroidSdk</Name>
     </ProjectReference>
   </ItemGroup>
-  <ItemGroup>
-    <None Include="packages.config" />
-  </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <Import Project="result-packaging.targets" />
-  <Import Project="..\..\packages\Xamarin.LibZipSharp.1.0.6\build\Xamarin.LibZipSharp.targets" Condition="Exists('..\..\packages\Xamarin.LibZipSharp.1.0.6\build\Xamarin.LibZipSharp.targets')" />
 </Project>

build-tools/Xamarin.Android.Tools.BootstrapTasks/packages.config

@@ -1,6 +1,6 @@
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
-  <Import Project="..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems" />
+  <Import Project="..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems" Condition="Exists('..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems')" />
 
   <PropertyGroup>
     <_TopDir>$(MSBuildThisFileDirectory)..\..</_TopDir>

build-tools/api-xml-adjuster/api-xml-adjuster.targets

@@ -21,7 +21,7 @@ variables:
   InstallerArtifactName: installers
   TestAssembliesArtifactName: test-assemblies
   NUnitConsoleVersion: 3.9.0
-  DotNetCoreVersion: 2.1.701
+  DotNetCoreVersion: 3.1.100
   HostedMacMojave: Hosted Mac Internal Mojave
   HostedMac: Hosted Mac Internal
   HostedWinVS2019: Hosted Windows 2019 with VS2019
@@ -83,23 +83,8 @@ stages:
       displayName: make jenkins
 
     # Build and package test assemblies
-    - task: MSBuild@1
-      displayName: msbuild tests/CodeBehind/UnitTests/CodeBehindUnitTests.csproj
-      inputs:
-        solution: $(System.DefaultWorkingDirectory)/tests/CodeBehind/UnitTests/CodeBehindUnitTests.csproj
-        configuration: $(XA.Build.Configuration)
-
-    - task: MSBuild@1
-      displayName: msbuild tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-UnitTests/Xamarin.Android.MakeBundle-UnitTests.csproj
-      inputs:
-        solution: $(System.DefaultWorkingDirectory)/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-UnitTests/Xamarin.Android.MakeBundle-UnitTests.csproj
-        configuration: $(XA.Build.Configuration)
-
-    - task: MSBuild@1
-      displayName: msbuild tests/EmbeddedDSOs/EmbeddedDSO-UnitTests/EmbeddedDSO-UnitTests.csproj
-      inputs:
-        solution: $(System.DefaultWorkingDirectory)/tests/EmbeddedDSOs/EmbeddedDSO-UnitTests/EmbeddedDSO-UnitTests.csproj
-        configuration: $(XA.Build.Configuration)
+    - script: make all-tests V=1 CONFIGURATION=$(XA.Build.Configuration)
+      displayName: make all-tests
 
     - script: |
         cp -r bin/$(XA.Build.Configuration)/bcl-tests bin/Test$(XA.Build.Configuration)/bcl-tests

build-tools/automation/azure-pipelines.yaml

@@ -174,6 +174,8 @@ timestamps {
                     }
                 }
             }
+            // Install .NET Core and temporarily append it to PATH
+            sh "curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -version 3.1.100"
         }
 
         utils.stageWithTimeout('build', 6, 'HOURS', XADir, true) {    // Typically takes less than one hour except a build on a new bot to populate local caches can take several hours

build-tools/automation/build.groovy

@@ -33,6 +33,10 @@ steps:
     version: $(DotNetCoreVersion)
 
 # Restore solutions for Xamarin.Android.Tools.sln, Xamarin.Android.sln, and Xamarin.Android-Tests.sln
+- task: NuGetToolInstaller@0
+  inputs:
+    versionSpec: 5.x
+
 - task: NuGetCommand@2
   displayName: nuget restore Xamarin.Android solutions
   inputs:

build-tools/automation/yaml-templates/setup-test-environment.yaml

@@ -2,8 +2,8 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="..\scripts\XAVersionInfo.targets" />
   <Import Project="..\scripts\LocalizationLanguages.projitems" />
-  <Import Project="..\..\bin\Build$(Configuration)\ProfileAssemblies.projitems" />
-  <Import Project="..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems" />
+  <Import Project="..\..\bin\Build$(Configuration)\ProfileAssemblies.projitems" Condition="Exists('..\..\bin\Build$(Configuration)\ProfileAssemblies.projitems')" />
+  <Import Project="..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems" Condition="Exists('..\..\bin\Build$(Configuration)\Mono.Android.Apis.projitems')" />
   <UsingTask AssemblyFile="..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.ReplaceFileContents" />
   <PropertyGroup>
     <RootBuildDir>$(XamarinAndroidSourcePath)bin\$(Configuration)\</RootBuildDir>

build-tools/installers/create-installers.targets

@@ -1,24 +1,23 @@
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <!--Import this file in projects needing to reference Microsoft.Build.*.dll -->
-  <PropertyGroup>
-    <MSBuildReferencePath Condition=" !$(MSBuildToolsPath.Contains('xbuild')) ">$(MSBuildToolsPath)</MSBuildReferencePath>
-    <MSBuildReferencePath Condition=" $(MSBuildToolsPath.Contains('xbuild')) ">$([System.IO.Path]::GetFullPath ('$(MSBuildToolsPath)\..\..\..\msbuild\15.0\bin'))</MSBuildReferencePath>
-  </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Microsoft.Build">
-      <HintPath>$(MSBuildReferencePath)\Microsoft.Build.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Build.Engine">
-      <HintPath>$(MSBuildReferencePath)\Microsoft.Build.Engine.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Build.Framework">
-      <HintPath>$(MSBuildReferencePath)\Microsoft.Build.Framework.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Build.Tasks.Core">
-      <HintPath>$(MSBuildReferencePath)\Microsoft.Build.Tasks.Core.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Build.Utilities.Core">
-      <HintPath>$(MSBuildReferencePath)\Microsoft.Build.Utilities.Core.dll</HintPath>
-    </Reference>
+    <!-- Compile against Microsoft.Build* NuGet refs, but do not copy to OutputDir. -->
+    <PackageReference Include="Microsoft.Build" Version="15.9.20" IncludeAssets="compile" />
+    <PackageReference Include="Microsoft.Build.Framework" Version="15.9.20" IncludeAssets="compile" />
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="15.9.20" IncludeAssets="compile" />
+    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="15.9.20" IncludeAssets="compile"/>
+    <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
+    <PackageReference Include="Xamarin.Build.AsyncTask" Version="0.3.4" />
+    <PackageReference Include="Xamarin.LibZipSharp" Version="1.0.6" />
   </ItemGroup>
-</Project>
+  <ItemGroup>
+    <!-- Copy system Microsoft.Build*.dll and dependencies for tests to run against. We can remove this
+          and rely entirely on NuGet assets when mono/msbuild is merged into microsoft/msbuild. -->
+    <None Include="$(MSBuildToolsPath)\Microsoft.Build*.dll" >
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="$(MSBuildToolsPath)\System.*.dll" >
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

build-tools/scripts/MSBuildReferences.projitems

@@ -6,7 +6,7 @@
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitsInRange" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitTime" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitInfo" />
-  <Import Project="..\..\bin\Build$(Configuration)\MonoGitHash.props" />
+  <Import Project="..\..\bin\Build$(Configuration)\MonoGitHash.props" Condition="Exists('..\..\bin\Build$(Configuration)\MonoGitHash.props')" />
   <Target Name="_GetSubmodulesVersionInfo">
     <ItemGroup>
       <!-- If anything changes in this list, make sure to update: