CLR hosting

CLR-Host-Notes.md

@@ -0,0 +1,42 @@
+# Notes
+
+## Potential optimizations
+
+  * https://github.com/dotnet/runtime/blob/9b24fb60a19f62620ca1fc5e4eb2e3ae0b3b086d/src/coreclr/binder/assemblybindercommon.cpp#L844-L889
+    * Managed C++ assemblies aren't available on Unix, no point in looking for them
+    * `candidates[]` is `WCHAR*`, while `ProbeAppBundle` takes UTF-8 - no point in doing the
+      conversion here
+  * Host contract
+    * It might make sense to pass strings as Unicode to host and back, to avoid conversions.
+      p/invoke names for instance, can be Unicode. So can be the assembly names. Native library
+      names should be UTF-8, but we can generate lookup tables for those at application build time
+      (e.g. indexed by an xxHash of the Unicode version of the name) - no conversion at run time.
+    * We need declarations of all he possible HRESULT errors (`S_OK` etc)
+
+## Stuff that should be changed
+
+### Logging
+Currently, most of the messages logged by the runtime end up in `/dev/null` (either because they
+are disabled in release build or because they log to stdio which doesn't work on Android).
+
+Logcat is the only way to get information from remote devices, especially via Google Play Console.
+
+We should log to logcat:
+
+    + C++ exception messages
+    + abort() messages / fatal errors
+    + warnings
+    + errors
+
+A subsystem should be added which will provide a single function that will do actual output, implementation of which
+will be specific to the platform.  API should allow specification of severity, the actual message, and possibly a flag
+to indicate whether the process should be aborted (the decision might also be based on the severity).  Severity should
+be shared between all targets, which then can (if needed) translate it to the target platform's value(s), if any.
+
+### Process termination
+Runtime currently calls `abort()` in several places.  This should probably become part of the host contract instead.
+Being part of the contract, the target platform could implement process termination on `abort()` in a uniform way
+(includes platform-specific logging, preparation etc)
+
+## Issues and workarounds
+

Configuration.props

@@ -220,4 +220,15 @@
     <AndroidAbiAndRuntimeFlavor Include="@(AndroidSupportedTargetJitAbi)" AndroidRuntime="NativeAOT" />
     <AndroidAbiAndRuntimeFlavor Include="@(AndroidSupportedTargetJitAbi)" AndroidRuntime="CoreCLR" />
   </ItemGroup>
+
+  <!-- Whenever there's a need to use a locally built CoreCLR, both .NET for Android and the
+       application(s) must be built with the same runtime. This property should point to the CoreCLR
+       artifact directory in its repository checkout:
+
+          {PATH_TO_DOTNET_RUNTIME_REPO}/artifacts/bin/microsoft.netcore.app.runtime.android-{ARCH}/{CONFIG}
+
+      <PropertyGroup>
+         <CLRLocalRuntimePath></CLRLocalRuntimePath>
+      </PropertyGroup>
+  -->
 </Project>

Xamarin.Android.sln

@@ -45,7 +45,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop.Tools.Diagnost
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop.Tools.Cecil", "external\Java.Interop\src\Java.Interop.Tools.Cecil\Java.Interop.Tools.Cecil.csproj", "{D48EE8D0-0A0A-4493-AEF5-DAF5F8CF86AD}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "native", "src\native\native-mono.csproj", "{53EE4C57-1C03-405A-8243-8DA539546C88}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "native-mono", "src\native\native-mono.csproj", "{53EE4C57-1C03-405A-8243-8DA539546C88}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{CAB438D8-B0F5-4AF0-BEBD-9E2ADBD7B483}"
 EndProject

build-tools/create-packs/Microsoft.Android.Runtime.proj

@@ -60,7 +60,7 @@ projects that use the Microsoft.Android framework in .NET 6+.
       <_AndroidRuntimePackAssets Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libunwind_xamarin.a" />
     </ItemGroup>
 
-    <ItemGroup Condition=" '$(AndroidRuntime)' == 'CoreCLR' ">
+    <ItemGroup Condition=" '$(AndroidRuntime)' != 'NativeAOT' ">
       <!-- TODO: the Exists() checks must go away once we build CoreCLR host for all the targets -->
       <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libc.so" />
       <_AndroidRuntimePackAssets Condition=" Exists('$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libdl.so') " Include="$(NativeRuntimeOutputRootDir)$(_RuntimeFlavorDirName)\$(AndroidRID)\libdl.so" />

build-tools/installers/create-installers.targets

@@ -133,6 +133,10 @@
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_fastdev_net6.jar" ExcludeFromLegacy="true" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_net6.dex" ExcludeFromLegacy="true" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_fastdev_net6.dex" ExcludeFromLegacy="true" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_clr.jar" ExcludeFromLegacy="true" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_fastdev_clr.jar" ExcludeFromLegacy="true" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_clr.dex" ExcludeFromLegacy="true" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)java_runtime_fastdev_clr.dex" ExcludeFromLegacy="true" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)manifestmerger.jar" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)proguard-android.txt" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)protobuf-net.dll" />
@@ -175,18 +179,6 @@
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)K4os.Compression.LZ4.dll" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ELFSharp.dll" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ManifestOverlays\Timing.xml" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libc.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libm.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libc.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libm.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libc.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libm.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libc.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libm.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-arm64\libarchive-dso-stub.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-arm\libarchive-dso-stub.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-x64\libarchive-dso-stub.so" />
-    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-x86\libarchive-dso-stub.so" />
   </ItemGroup>
   <ItemGroup>
     <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" />

build-tools/scripts/generate-pinvoke-tables.sh

@@ -2,11 +2,14 @@
 MY_DIR="$(dirname $0)"
 HOST="$(uname | tr A-Z a-z)"
 
-NATIVE_DIR="${MY_DIR}/../../src/native/mono"
-MONODROID_SOURCE_DIR="${NATIVE_DIR}/pinvoke-override"
-GENERATOR_SOURCE="${MONODROID_SOURCE_DIR}/generate-pinvoke-tables.cc"
-GENERATOR_BINARY="${MONODROID_SOURCE_DIR}/generate-pinvoke-tables"
-TARGET_FILE="${MONODROID_SOURCE_DIR}/pinvoke-tables.include"
+NATIVE_DIR="${MY_DIR}/../../src/native"
+MONODROID_SOURCE_DIR="${NATIVE_DIR}/mono/pinvoke-override"
+MONODROID_INCLUDE_DIR="${NATIVE_DIR}/mono/shared"
+CLR_SOURCE_DIR="${NATIVE_DIR}/clr/host"
+CLR_INCLUDE_DIR="${NATIVE_DIR}/clr/include/shared"
+GENERATOR_SOURCE="generate-pinvoke-tables.cc"
+GENERATOR_BINARY="generate-pinvoke-tables"
+TARGET_FILE="pinvoke-tables.include"
 GENERATED_FILE="${TARGET_FILE}.generated"
 DIFF_FILE="${TARGET_FILE}.diff"
 EXTERNAL_DIR="${MY_DIR}/../../external/"
@@ -64,33 +67,61 @@ case ${HOST} in
 	*) die Unsupported OS ;;
 esac
 
-${COMPILER} -O2 -std=c++20 -I${EXTERNAL_DIR} -I${EXTERNAL_DIR}/constexpr-xxh3 -I${NATIVE_DIR}/shared -I${NATIVE_DIR}/../common/include "${GENERATOR_SOURCE}" -o "${GENERATOR_BINARY}"
-"${GENERATOR_BINARY}" "${GENERATED_FILE}"
-
-FILES_DIFFER="no"
-cmp "${GENERATED_FILE}" "${TARGET_FILE}" > /dev/null 2>&1 || FILES_DIFFER="yes"
-
-RETVAL=0
-if [ "${TEST_ONLY}" == "no" ]; then
-	if [ "${FILES_DIFFER}" == "yes" ]; then
-		mv "${GENERATED_FILE}" "${TARGET_FILE}"
-	else
-		rm "${GENERATED_FILE}"
-	fi
-else
-	if [ "${FILES_DIFFER}" == "yes" ]; then
-		echo "Generated p/invokes table file differs from the current one"
-		diff -U3 -Narp "${TARGET_FILE}" "${GENERATED_FILE}" > "${DIFF_FILE}"
-
-		echo "Diff file saved in: ${DIFF_FILE}"
-		echo "------ DIFF START ------"
-		cat "${DIFF_FILE}"
-		echo "------ DIFF END ------"
-		echo
-		RETVAL=1
+function generate()
+{
+	local SOURCE_DIR="${1}"
+	local INCLUDE_DIR="${2}"
+	local SOURCE="${SOURCE_DIR}/${GENERATOR_SOURCE}"
+	local BINARY="${SOURCE_DIR}/${GENERATOR_BINARY}"
+	local RESULT="${SOURCE_DIR}/${GENERATED_FILE}"
+	local TARGET="${SOURCE_DIR}/${TARGET_FILE}"
+	local DIFF="${SOURCE_DIR}/${DIFF_FILE}"
+
+	${COMPILER} -O2 -std=c++20 -I${EXTERNAL_DIR} -I${EXTERNAL_DIR}/constexpr-xxh3 -I${INCLUDE_DIR} -I${NATIVE_DIR}/common/include "${SOURCE}" -o "${BINARY}"
+	"${BINARY}" "${RESULT}"
+
+	FILES_DIFFER="no"
+	cmp "${RESULT}" "${TARGET}" > /dev/null 2>&1 || FILES_DIFFER="yes"
+
+	if [ "${TEST_ONLY}" == "no" ]; then
+		if [ "${FILES_DIFFER}" == "yes" ]; then
+			  mv "${RESULT}" "${TARGET}"
+		else
+			rm "${RESULT}"
+		fi
 	else
-		echo Generated file is identical to the current one
+		if [ "${FILES_DIFFER}" == "yes" ]; then
+			echo "Generated p/invokes table file differs from the current one"
+			diff -U3 -Narp "${TARGET}" "${RESULT}" > "${DIFF}"
+
+			echo "Diff file saved in: ${DIFF}"
+			echo "------ DIFF START ------"
+			cat "${DIFF}"
+			echo "------ DIFF END ------"
+			echo
+			RETVAL=1
+		else
+			echo Generated file is identical to the current one
+		fi
 	fi
-fi
+}
+
+RETVAL=0
+cat <<EOF
+**
+** Generating for MonoVM
+**
+EOF
+generate "${MONODROID_SOURCE_DIR}" "${MONODROID_INCLUDE_DIR}"
+
+cat <<EOF
+
+--------------------------------------
+
+**
+** Generating for CoreCLR
+**
+EOF
+generate "${CLR_SOURCE_DIR}" "${CLR_INCLUDE_DIR}"
 
 exit ${RETVAL}

build-tools/scripts/xa_build_configuration.cmake.in

@@ -2,3 +2,8 @@ set(NETCORE_APP_RUNTIME_DIR_ARM "@NETCORE_APP_RUNTIME_ANDROID_ARM@")
 set(NETCORE_APP_RUNTIME_DIR_ARM64 "@NETCORE_APP_RUNTIME_ANDROID_ARM64@")
 set(NETCORE_APP_RUNTIME_DIR_X86 "@NETCORE_APP_RUNTIME_ANDROID_X86@")
 set(NETCORE_APP_RUNTIME_DIR_X86_64 "@NETCORE_APP_RUNTIME_ANDROID_X86_64@")
+
+set(CORECLR_APP_RUNTIME_DIR_ARM "@CORECLR_APP_RUNTIME_ANDROID_ARM@")
+set(CORECLR_APP_RUNTIME_DIR_ARM64 "@CORECLR_APP_RUNTIME_ANDROID_ARM64@")
+set(CORECLR_APP_RUNTIME_DIR_X86 "@CORECLR_APP_RUNTIME_ANDROID_X86@")
+set(CORECLR_APP_RUNTIME_DIR_X86_64 "@CORECLR_APP_RUNTIME_ANDROID_X86_64@")

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs

@@ -191,12 +191,18 @@ public static partial class Paths
 			public static string OpenJDKInstallDir                   => GetCachedPath (ref openJDKInstallDir, ()                   => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), Defaults.JdkFolder));
 			public static string OpenJDKCacheDir                     => GetCachedPath (ref openJDKCacheDir, ()                     => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory));
 
-			// .NET 6
+			// .NET 6+
 			public static string NetcoreAppRuntimeAndroidARM         => GetCachedPath (ref netcoreAppRuntimeAndroidARM, () => GetNetcoreAppRuntimePath (ctx, "arm"));
 			public static string NetcoreAppRuntimeAndroidARM64       => GetCachedPath (ref netcoreAppRuntimeAndroidARM64, () => GetNetcoreAppRuntimePath (ctx, "arm64"));
 			public static string NetcoreAppRuntimeAndroidX86         => GetCachedPath (ref netcoreAppRuntimeAndroidX86, () => GetNetcoreAppRuntimePath (ctx, "x86"));
 			public static string NetcoreAppRuntimeAndroidX86_64      => GetCachedPath (ref netcoreAppRuntimeAndroidX86_64, () => GetNetcoreAppRuntimePath (ctx, "x64"));
 
+			// CoreCLR
+			public static string CoreClrAppRuntimeAndroidARM         => GetCachedPath (ref coreclrAppRuntimeAndroidARM, () => GetCoreClrAppRuntimePath (ctx, "arm"));
+			public static string CoreClrAppRuntimeAndroidARM64       => GetCachedPath (ref coreclrAppRuntimeAndroidARM64, () => GetCoreClrAppRuntimePath (ctx, "arm64"));
+			public static string CoreClrAppRuntimeAndroidX86         => GetCachedPath (ref coreclrAppRuntimeAndroidX86, () => GetCoreClrAppRuntimePath (ctx, "x86"));
+			public static string CoreClrAppRuntimeAndroidX86_64      => GetCachedPath (ref coreclrAppRuntimeAndroidX86_64, () => GetCoreClrAppRuntimePath (ctx, "x64"));
+
 			public static string MicrosoftNETWorkloadMonoPackageDir => Path.Combine (
 				XAPackagesDir,
 				$"microsoft.net.workload.mono.toolchain.{{0}}.manifest-{ctx.Properties.GetRequiredValue (KnownProperties.DotNetMonoManifestVersionBand)}",
@@ -242,6 +248,17 @@ static string GetNetcoreAppRuntimePath (Context ctx, string androidTarget)
 				);
 			}
 
+			static string GetCoreClrAppRuntimePath (Context ctx, string androidTarget)
+			{
+				return Path.Combine (
+					XAPackagesDir,
+					$"microsoft.netcore.app.runtime.android-{androidTarget}",
+					ctx.Properties.GetRequiredValue (KnownProperties.MicrosoftNETCoreAppRefPackageVersion),
+					"runtimes",
+					$"android-{androidTarget}"
+				);
+			}
+
 			static string EnsureAndroidToolchainBinDirectories ()
 			{
 				if (androidToolchainBinDirectory != null)
@@ -278,6 +295,10 @@ static string GetCachedPath (ref string? variable, Func<string> creator)
 			static string? netcoreAppRuntimeAndroidARM64;
 			static string? netcoreAppRuntimeAndroidX86;
 			static string? netcoreAppRuntimeAndroidX86_64;
+			static string? coreclrAppRuntimeAndroidARM;
+			static string? coreclrAppRuntimeAndroidARM64;
+			static string? coreclrAppRuntimeAndroidX86;
+			static string? coreclrAppRuntimeAndroidX86_64;
 		}
 	}
 }

build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs

@@ -98,6 +98,11 @@ GeneratedFile Get_Cmake_XA_Build_Configuration (Context context)
 				{ "@NETCORE_APP_RUNTIME_ANDROID_ARM64@",  Utilities.EscapePathSeparators (Configurables.Paths.NetcoreAppRuntimeAndroidARM64) },
 				{ "@NETCORE_APP_RUNTIME_ANDROID_X86@",    Utilities.EscapePathSeparators (Configurables.Paths.NetcoreAppRuntimeAndroidX86) },
 				{ "@NETCORE_APP_RUNTIME_ANDROID_X86_64@", Utilities.EscapePathSeparators (Configurables.Paths.NetcoreAppRuntimeAndroidX86_64) },
+
+				{ "@CORECLR_APP_RUNTIME_ANDROID_ARM@",    Utilities.EscapePathSeparators (Configurables.Paths.CoreClrAppRuntimeAndroidARM) },
+				{ "@CORECLR_APP_RUNTIME_ANDROID_ARM64@",  Utilities.EscapePathSeparators (Configurables.Paths.CoreClrAppRuntimeAndroidARM64) },
+				{ "@CORECLR_APP_RUNTIME_ANDROID_X86@",    Utilities.EscapePathSeparators (Configurables.Paths.CoreClrAppRuntimeAndroidX86) },
+				{ "@CORECLR_APP_RUNTIME_ANDROID_X86_64@", Utilities.EscapePathSeparators (Configurables.Paths.CoreClrAppRuntimeAndroidX86_64) },
 			};
 
 			return new GeneratedPlaceholdersFile (
@@ -107,7 +112,7 @@ GeneratedFile Get_Cmake_XA_Build_Configuration (Context context)
 			);
 		}
 
-		GeneratedFile Get_Cmake_Presets (Context context)
+		GeneratedFile GetCmakePresetsCommon (Context context, string sourcesDir)
 		{
 			const string OutputFileName = "CMakePresets.json";
 
@@ -126,11 +131,16 @@ GeneratedFile Get_Cmake_Presets (Context context)
 
 			return new GeneratedPlaceholdersFile (
 				replacements,
-				Path.Combine (Configurables.Paths.NativeSourcesDir, $"{OutputFileName}.in"),
-				Path.Combine (Configurables.Paths.NativeSourcesDir, OutputFileName)
+				Path.Combine (sourcesDir, $"{OutputFileName}.in"),
+				Path.Combine (sourcesDir, OutputFileName)
 			);
 		}
 
+		GeneratedFile Get_Cmake_Presets (Context context)
+		{
+			return GetCmakePresetsCommon (context, Configurables.Paths.NativeSourcesDir);
+		}
+
 		GeneratedFile Get_Configuration_Generated_Props (Context context)
 		{
 			const string OutputFileName = "Configuration.Generated.props";

build-tools/xaprepare/xaprepare/package-download.proj

@@ -20,6 +20,8 @@ Otherwise, $(MicrosoftNETCoreAppRefPackageVersion) from eng/Versions.props will
     <PackageDownload Include="Microsoft.NETCore.App.Runtime.Mono.android-arm64" Version="[$(DotNetRuntimePacksVersion)]" />
     <PackageDownload Include="Microsoft.NETCore.App.Runtime.Mono.android-x86" Version="[$(DotNetRuntimePacksVersion)]" />
     <PackageDownload Include="Microsoft.NETCore.App.Runtime.Mono.android-x64" Version="[$(DotNetRuntimePacksVersion)]" />
+    <PackageDownload Include="Microsoft.NETCore.App.Runtime.android-arm64" Version="[$(DotNetRuntimePacksVersion)]" />
+    <PackageDownload Include="Microsoft.NETCore.App.Runtime.android-x64" Version="[$(DotNetRuntimePacksVersion)]" />
     <PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest-$(DotNetMonoManifestVersionBand)" Version="[$(DotNetRuntimePacksVersion)]" />
     <PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest-$(DotNetMonoManifestVersionBand)" Version="[$(DotNetRuntimePacksVersion)]" />
     <PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest-$(DotNetMonoManifestVersionBand)" Version="[$(DotNetRuntimePacksVersion)]" />

src-ThirdParty/dotnet/runtime/README

@@ -0,0 +1,11 @@
+The header files in this directory are verbatim copies taken from the
+https://github.com/dotnet/runtime/ repository. 
+
+They can be found in the following locations in the repository:
+
+  src/native/corehost/host_runtime_contract.h
+  src/coreclr/hosts/inc/coreclrhost.h
+
+and they MUST be in sync with the current version of the runtime being 
+used. They don't have to come from the same version, but their content 
+must be ABI and API compatible with it.