Skip to content

Commit faaad5a

Browse files
jpobstjpobst
committed
[XABT] Move JLO scanning needed for typemap generation to a linker step.
1 parent b670998 commit faaad5a

File tree

8 files changed

+709
-61
lines changed

8 files changed

+709
-61
lines changed

‎src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/FindTypeMapObjectsStep.cs‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#nullable enable
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Linq;
5+
using Microsoft.Android.Build.Tasks;
6+
using Microsoft.Build.Utilities;
7+
using Mono.Cecil;
8+
using Mono.Linker;
9+
using Mono.Linker.Steps;
10+
using Xamarin.Android.Tasks;
11+
12+
namespace MonoDroid.Tuner;
13+
14+
/// <summary>
15+
/// Scans an assembly for JLOs that need to be in the typemap and writes them to an XML file.
16+
/// </summary>
17+
public class FindTypeMapObjectsStep : BaseStep, IAssemblyModifierPipelineStep
18+
{
19+
public bool Debug { get; set; }
20+
21+
public bool ErrorOnCustomJavaObject { get; set; }
22+
23+
public TaskLoggingHelper Log { get; set; }
24+
25+
public FindTypeMapObjectsStep (TaskLoggingHelper log) => Log = log;
26+
27+
public bool ProcessAssembly (AssemblyDefinition assembly, StepContext context)
28+
{
29+
var destinationTypeMapXml = TypeMapObjectsXmlFile.GetTypeMapObjectsXmlFilePath (context.Destination.ItemSpec);
30+
31+
// We only care about assemblies that can contains JLOs
32+
if (!context.IsAndroidAssembly) {
33+
Log.LogDebugMessage ($"Skipping assembly '{assembly.Name.Name}' because it is not an Android assembly");
34+
TypeMapObjectsXmlFile.WriteEmptyFile (destinationTypeMapXml, Log);
35+
return false;
36+
}
37+
38+
var types = ScanForJavaTypes (assembly);
39+
40+
var xml = new TypeMapObjectsXmlFile {
41+
AssemblyName = assembly.Name.Name,
42+
};
43+
44+
if (Debug) {
45+
var (javaToManaged, managedToJava) = TypeMapCecilAdapter.GetDebugNativeEntries (types, Context, out var foundJniNativeRegistration);
46+
47+
xml.JavaToManagedDebugEntries.AddRange (javaToManaged);
48+
xml.ManagedToJavaDebugEntries.AddRange (managedToJava);
49+
xml.FoundJniNativeRegistration = foundJniNativeRegistration;
50+
51+
if (!xml.HasDebugEntries) {
52+
Log.LogDebugMessage ($"No Java types found in '{assembly.Name.Name}'");
53+
TypeMapObjectsXmlFile.WriteEmptyFile (destinationTypeMapXml, Log);
54+
return false;
55+
}
56+
} else {
57+
var genState = TypeMapCecilAdapter.GetReleaseGenerationState (types, Context, out var foundJniNativeRegistration);
58+
xml.ModuleReleaseData = genState.TempModules.SingleOrDefault ().Value;
59+
60+
if (xml.ModuleReleaseData == null) {
61+
Log.LogDebugMessage ($"No Java types found in '{assembly.Name.Name}'");
62+
TypeMapObjectsXmlFile.WriteEmptyFile (destinationTypeMapXml, Log);
63+
return false;
64+
}
65+
}
66+
67+
xml.Export (destinationTypeMapXml);
68+
69+
Log.LogDebugMessage ($"Wrote '{destinationTypeMapXml}', {xml.JavaToManagedDebugEntries.Count} JavaToManagedDebugEntries, {xml.ManagedToJavaDebugEntries.Count} ManagedToJavaDebugEntries, FoundJniNativeRegistration: {xml.FoundJniNativeRegistration}");
70+
71+
// This step does not change the assembly
72+
return false;
73+
}
74+
75+
List<TypeDefinition> ScanForJavaTypes (AssemblyDefinition assembly)
76+
{
77+
var types = new List<TypeDefinition> ();
78+
79+
var scanner = new XAJavaTypeScanner (Xamarin.Android.Tools.AndroidTargetArch.None, Log, Context) {
80+
ErrorOnCustomJavaObject = ErrorOnCustomJavaObject
81+
};
82+
83+
foreach (ModuleDefinition md in assembly.Modules) {
84+
foreach (TypeDefinition td in md.Types) {
85+
scanner.AddJavaType (td, types);
86+
}
87+
}
88+
89+
return types;
90+
}
91+
}

‎src/Xamarin.Android.Build.Tasks/Tasks/AssemblyModifierPipeline.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,15 @@ protected virtual void BuildPipeline (AssemblyPipeline pipeline, MSBuildLinkCont
141141

142142
findJavaObjectsStep.Initialize (context);
143143
pipeline.Steps.Add (findJavaObjectsStep);
144+
145+
// FindTypeMapObjectsStep
146+
var findTypeMapObjectsStep = new FindTypeMapObjectsStep (Log) {
147+
ErrorOnCustomJavaObject = ErrorOnCustomJavaObject,
148+
Debug = Debug,
149+
};
150+
151+
findTypeMapObjectsStep.Initialize (context);
152+
pipeline.Steps.Add (findTypeMapObjectsStep);
144153
}
145154

146155
void RunPipeline (AssemblyPipeline pipeline, ITaskItem source, ITaskItem destination, WriterParameters writerParameters)

‎src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMappings.cs‎

Lines changed: 85 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
using System;
33
using System.Collections.Concurrent;
44
using System.Collections.Generic;
5+
using System.Diagnostics;
56
using System.IO;
7+
using System.Linq;
68
using Java.Interop.Tools.Cecil;
79
using Microsoft.Android.Build.Tasks;
810
using Microsoft.Build.Framework;
@@ -20,67 +22,127 @@ public class GenerateTypeMappings : AndroidTask
2022

2123
public bool Debug { get; set; }
2224

25+
public bool EnableMarshalMethods { get;set; }
26+
27+
[Output]
28+
public ITaskItem [] GeneratedBinaryTypeMaps { get; set; } = [];
29+
2330
[Required]
2431
public string IntermediateOutputDirectory { get; set; } = "";
2532

2633
public bool SkipJniAddNativeMethodRegistrationAttributeScan { get; set; }
2734

2835
[Required]
29-
public string TypemapOutputDirectory { get; set; } = "";
36+
public ITaskItem [] ResolvedAssemblies { get; set; } = [];
3037

31-
[Output]
32-
public ITaskItem [] GeneratedBinaryTypeMaps { get; set; } = [];
38+
// This property is temporary and is used to ensure that the new "linker step"
39+
// JLO scanning produces the same results as the old process. It will be removed
40+
// once the process is complete.
41+
public bool RunCheckedBuild { get; set; }
42+
43+
[Required]
44+
public string [] SupportedAbis { get; set; } = [];
3345

3446
public string TypemapImplementation { get; set; } = "llvm-ir";
3547

48+
[Required]
49+
public string TypemapOutputDirectory { get; set; } = "";
50+
3651
AndroidRuntime androidRuntime;
3752

3853
public override bool RunTask ()
3954
{
55+
var useMarshalMethods = !Debug && EnableMarshalMethods;
56+
4057
androidRuntime = MonoAndroidHelper.ParseAndroidRuntime (AndroidRuntime);
58+
if (androidRuntime == Xamarin.Android.Tasks.AndroidRuntime.NativeAOT) {
59+
// NativeAOT typemaps are generated in `Microsoft.Android.Sdk.ILLink.TypeMappingStep`
60+
Log.LogDebugMessage ("Skipping type maps for NativeAOT.");
61+
return !Log.HasLoggedErrors;
62+
}
4163

42-
// Retrieve the stored NativeCodeGenState
43-
var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
44-
MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
45-
RegisteredTaskObjectLifetime.Build
46-
);
64+
// If using marshal methods, we cannot use the .typemap.xml files currently because
65+
// the type token ids were changed by the marshal method rewriter.
66+
if (!useMarshalMethods)
67+
GenerateAllTypeMappings ();
4768

48-
NativeCodeGenState? templateCodeGenState = null;
69+
// Temporarily used to ensure we still generate the same as the old code
70+
if (RunCheckedBuild || useMarshalMethods) {
71+
// Retrieve the stored NativeCodeGenState
72+
var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
73+
MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
74+
RegisteredTaskObjectLifetime.Build
75+
);
4976

50-
foreach (var kvp in nativeCodeGenStates) {
51-
NativeCodeGenState state = kvp.Value;
52-
templateCodeGenState = state;
53-
WriteTypeMappings (state);
54-
}
77+
NativeCodeGenState? templateCodeGenState = null;
78+
79+
foreach (var kvp in nativeCodeGenStates) {
80+
NativeCodeGenState state = kvp.Value;
81+
templateCodeGenState = state;
82+
WriteTypeMappingsFromNativeState (state, useMarshalMethods);
83+
}
84+
85+
if (templateCodeGenState is null)
86+
throw new InvalidOperationException ($"Internal error: no native code generator state defined");
5587

56-
if (templateCodeGenState is null)
57-
throw new InvalidOperationException ($"Internal error: no native code generator state defined");
88+
// Set for use by <GeneratePackageManagerJava/> task later
89+
NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent = templateCodeGenState.JniAddNativeMethodRegistrationAttributePresent;
5890

59-
// Set for use by <GeneratePackageManagerJava/> task later
60-
NativeCodeGenState.TemplateJniAddNativeMethodRegistrationAttributePresent = templateCodeGenState.JniAddNativeMethodRegistrationAttributePresent;
91+
return !Log.HasLoggedErrors;
92+
}
6193

6294
return !Log.HasLoggedErrors;
6395
}
6496

65-
void WriteTypeMappings (NativeCodeGenState state)
97+
void GenerateAllTypeMappings ()
98+
{
99+
var allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, SupportedAbis, validate: true);
100+
101+
foreach (var set in allAssembliesPerArch)
102+
GenerateTypeMap (set.Key, set.Value.Values.ToList ());
103+
}
104+
105+
void GenerateTypeMap (AndroidTargetArch arch, List<ITaskItem> assemblies)
106+
{
107+
Log.LogDebugMessage ($"Generating type maps for architecture '{arch}'");
108+
109+
var state = TypeMapObjectsFileAdapter.Create (arch, assemblies, Log);
110+
111+
// An error was already logged to Log.LogError
112+
if (state is null)
113+
return;
114+
115+
var tmg = new TypeMapGenerator (Log, state, androidRuntime);
116+
tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, TypemapOutputDirectory);
117+
118+
AddOutputTypeMaps (tmg, state.TargetArch);
119+
}
120+
121+
void WriteTypeMappingsFromNativeState (NativeCodeGenState state, bool useMarshalMethods)
66122
{
67123
if (androidRuntime == Xamarin.Android.Tasks.AndroidRuntime.NativeAOT) {
68124
// NativeAOT typemaps are generated in `Microsoft.Android.Sdk.ILLink.TypeMappingStep`
69125
Log.LogDebugMessage ("Skipping type maps for NativeAOT.");
70126
return;
71127
}
72-
Log.LogDebugMessage ($"Generating type maps for architecture '{state.TargetArch}'");
128+
Log.LogDebugMessage ($"Generating type maps from native state for architecture '{state.TargetArch}' (RunCheckedBuild = {RunCheckedBuild})");
73129

74130
if (TypemapImplementation != "llvm-ir") {
75131
Log.LogDebugMessage ($"TypemapImplementation='{TypemapImplementation}' will write an empty native typemap.");
76132
state = new NativeCodeGenState (state.TargetArch, new TypeDefinitionCache (), state.Resolver, [], [], state.Classifier);
77133
}
78134

79-
var tmg = new TypeMapGenerator (Log, state, androidRuntime);
135+
var tmg = new TypeMapGenerator (Log, new NativeCodeGenStateAdapter (state), androidRuntime) { RunCheckedBuild = RunCheckedBuild && !useMarshalMethods };
80136
tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, TypemapOutputDirectory);
81137

82-
string abi = MonoAndroidHelper.ArchToAbi (state.TargetArch);
138+
AddOutputTypeMaps (tmg, state.TargetArch);
139+
}
140+
141+
void AddOutputTypeMaps (TypeMapGenerator tmg, AndroidTargetArch arch)
142+
{
143+
string abi = MonoAndroidHelper.ArchToAbi (arch);
83144
var items = new List<ITaskItem> ();
145+
84146
foreach (string file in tmg.GeneratedBinaryTypeMaps) {
85147
var item = new TaskItem (file);
86148
string fileName = Path.GetFileName (file);
@@ -90,6 +152,6 @@ void WriteTypeMappings (NativeCodeGenState state)
90152
items.Add (item);
91153
}
92154

93-
GeneratedBinaryTypeMaps = items.ToArray ();
155+
GeneratedBinaryTypeMaps = GeneratedBinaryTypeMaps.Concat (items).ToArray ();
94156
}
95157
}

‎src/Xamarin.Android.Build.Tasks/Utilities/TypeMapCecilAdapter.cs‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,25 @@ class TypeMapCecilAdapter
1616
{
1717
public static (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries (NativeCodeGenState state)
1818
{
19+
var (javaToManaged, managedToJava) = GetDebugNativeEntries (state.AllJavaTypes, state.TypeCache, out var foundJniNativeRegistration);
20+
21+
state.JniAddNativeMethodRegistrationAttributePresent = foundJniNativeRegistration;
22+
23+
return (javaToManaged, managedToJava);
24+
}
25+
26+
public static (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries (List<TypeDefinition> types, TypeDefinitionCache cache, out bool foundJniNativeRegistration)
27+
{
28+
var javaDuplicates = new Dictionary<string, List<TypeMapDebugEntry>> (StringComparer.Ordinal);
1929
var javaToManaged = new List<TypeMapDebugEntry> ();
2030
var managedToJava = new List<TypeMapDebugEntry> ();
31+
foundJniNativeRegistration = false;
2132

22-
var javaDuplicates = new Dictionary<string, List<TypeMapDebugEntry>> (StringComparer.Ordinal);
23-
foreach (TypeDefinition td in state.AllJavaTypes) {
24-
UpdateApplicationConfig (state, td);
33+
foreach (var td in types) {
34+
foundJniNativeRegistration = JniAddNativeMethodRegistrationAttributeFound (foundJniNativeRegistration, td);
2535

26-
TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache);
27-
HandleDebugDuplicates (javaDuplicates, entry, td, state.TypeCache);
36+
TypeMapDebugEntry entry = GetDebugEntry (td, cache);
37+
HandleDebugDuplicates (javaDuplicates, entry, td, cache);
2838

2939
javaToManaged.Add (entry);
3040
managedToJava.Add (entry);
@@ -40,16 +50,29 @@ public static ReleaseGenerationState GetReleaseGenerationState (NativeCodeGenSta
4050
var genState = new ReleaseGenerationState ();
4151

4252
foreach (TypeDefinition td in state.AllJavaTypes) {
43-
ProcessReleaseType (state, genState, td);
53+
UpdateApplicationConfig (state, td);
54+
ProcessReleaseType (state.TypeCache, genState, td);
4455
}
4556

4657
return genState;
4758
}
4859

49-
static void ProcessReleaseType (NativeCodeGenState state, ReleaseGenerationState genState, TypeDefinition td)
60+
public static ReleaseGenerationState GetReleaseGenerationState (List<TypeDefinition> types, TypeDefinitionCache cache, out bool foundJniNativeRegistration)
5061
{
51-
UpdateApplicationConfig (state, td);
52-
genState.AddKnownAssembly (GetAssemblyName (td));
62+
var genState = new ReleaseGenerationState ();
63+
foundJniNativeRegistration = false;
64+
65+
foreach (TypeDefinition td in types) {
66+
foundJniNativeRegistration = JniAddNativeMethodRegistrationAttributeFound (foundJniNativeRegistration, td);
67+
ProcessReleaseType (cache, genState, td);
68+
}
69+
70+
return genState;
71+
}
72+
73+
static void ProcessReleaseType (TypeDefinitionCache cache, ReleaseGenerationState genState, TypeDefinition td)
74+
{
75+
//genState.AddKnownAssembly (GetAssemblyName (td));
5376

5477
// We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding
5578
// byte array representation and on the runtime we need the latter in order to be able to binary search
@@ -74,7 +97,7 @@ static void ProcessReleaseType (NativeCodeGenState state, ReleaseGenerationState
7497
tempModules.Add (moduleUUID, moduleData);
7598
}
7699

77-
string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, state.TypeCache);
100+
string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache);
78101
// We will ignore generic types and interfaces when generating the Java to Managed map, but we must not
79102
// omit them from the table we output - we need the same number of entries in both java-to-managed and
80103
// managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator
@@ -147,6 +170,7 @@ static void HandleDebugDuplicates (Dictionary<string, List<TypeMapDebugEntry>> j
147170
// Fix things up so the abstract type is first, and the `Invoker` is considered a duplicate.
148171
duplicates.Insert (0, entry);
149172
oldEntry.SkipInJavaToManaged = false;
173+
oldEntry.IsInvoker = true;
150174
} else {
151175
// ¯\_(ツ)_/¯
152176
duplicates.Add (entry);
@@ -179,15 +203,20 @@ static void SyncDebugDuplicates (Dictionary<string, List<TypeMapDebugEntry>> jav
179203

180204
static void UpdateApplicationConfig (NativeCodeGenState state, TypeDefinition javaType)
181205
{
182-
if (state.JniAddNativeMethodRegistrationAttributePresent || !javaType.HasCustomAttributes) {
183-
return;
184-
}
206+
state.JniAddNativeMethodRegistrationAttributePresent = JniAddNativeMethodRegistrationAttributeFound (state.JniAddNativeMethodRegistrationAttributePresent, javaType);
207+
}
185208

209+
static bool JniAddNativeMethodRegistrationAttributeFound (bool alreadyFound, TypeDefinition javaType)
210+
{
211+
if (alreadyFound || !javaType.HasCustomAttributes) {
212+
return alreadyFound;
213+
}
214+
186215
foreach (CustomAttribute ca in javaType.CustomAttributes) {
187-
if (!state.JniAddNativeMethodRegistrationAttributePresent && String.Compare ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal) == 0) {
188-
state.JniAddNativeMethodRegistrationAttributePresent = true;
189-
break;
216+
if (string.Equals ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal)) {
217+
return true;
190218
}
191219
}
220+
return false;
192221
}
193222
}

0 commit comments

Comments
 (0)