From 2d283ae1c95ab9453624c0052b9389dd39dfcc82 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 3 Jul 2025 16:01:30 +0200 Subject: [PATCH 01/12] Scaffolding for the new app --- tools/apput/projects/.placeholder | 1 + .../ApplicationAssembly.cs | 25 +++++++ .../apput/src/AssemblyStore/AssemblyStore.cs | 24 +++++++ tools/apput/src/Common/IAspect.cs | 32 +++++++++ tools/apput/src/Common/NativeArchitecture.cs | 11 +++ tools/apput/src/Native/SharedLibrary.cs | 68 +++++++++++++++++++ tools/apput/src/Package/ApplicationPackage.cs | 21 ++++++ tools/apput/src/Package/PackageAAB.cs | 6 ++ tools/apput/src/Package/PackageAPK.cs | 6 ++ tools/apput/src/Package/PackageBase.cs | 6 ++ tools/apput/src/Program.cs | 8 +++ tools/apput/src/apput.csproj | 28 ++++++++ 12 files changed, 236 insertions(+) create mode 100644 tools/apput/projects/.placeholder create mode 100644 tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStore.cs create mode 100644 tools/apput/src/Common/IAspect.cs create mode 100644 tools/apput/src/Common/NativeArchitecture.cs create mode 100644 tools/apput/src/Native/SharedLibrary.cs create mode 100644 tools/apput/src/Package/ApplicationPackage.cs create mode 100644 tools/apput/src/Package/PackageAAB.cs create mode 100644 tools/apput/src/Package/PackageAPK.cs create mode 100644 tools/apput/src/Package/PackageBase.cs create mode 100644 tools/apput/src/Program.cs create mode 100644 tools/apput/src/apput.csproj diff --git a/tools/apput/projects/.placeholder b/tools/apput/projects/.placeholder new file mode 100644 index 00000000000..a18be027f41 --- /dev/null +++ b/tools/apput/projects/.placeholder @@ -0,0 +1 @@ +This directory will contain .csproj files to be used by 3rd parties (e.g. XABT tests) diff --git a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs new file mode 100644 index 00000000000..37427304ca2 --- /dev/null +++ b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +public class ApplicationAssembly : IAspect +{ + public static string AspectName { get; } = "Application assembly"; + + public bool IsCompressed { get; private set; } + public string Name { get; private set; } = ""; + public ulong CompressedSize { get; private set; } + public ulong Size { get; private set; } + public bool IgnoreOnLoad { get; private set; } + + public static IAspect LoadAspect (Stream stream, string description) + { + throw new NotImplementedException (); + } + + public static bool ProbeAspect (Stream stream) + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs new file mode 100644 index 00000000000..32c7f96efa4 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace ApplicationUtility; + +public class AssemblyStore : IAspect +{ + public static string AspectName { get; } = "Assembly Store"; + + public IDictionary Assemblies { get; private set; } = null!; + public NativeArchitecture Architecture { get; private set; } = NativeArchitecture.Unknown; + public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); + + public static IAspect LoadAspect (Stream stream, string description) + { + throw new NotImplementedException (); + } + + public static bool ProbeAspect (Stream stream) + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/Common/IAspect.cs b/tools/apput/src/Common/IAspect.cs new file mode 100644 index 00000000000..c078a5e9fb2 --- /dev/null +++ b/tools/apput/src/Common/IAspect.cs @@ -0,0 +1,32 @@ +using System.IO; + +namespace ApplicationUtility; + +/// +/// Represets an aspect of a .NET for Android application. An aspect can be an +/// individual assembly, the whole APK/AAB package, a shared library etc. +/// If it exists as a definable, separate entity in the application, that can +/// be identified/detected by looking at its format/location it is most +/// likely an aspect. +/// +public interface IAspect +{ + /// + /// Aspect name, for presentation purposes. + /// + abstract static string AspectName { get; } + + /// + /// Probes whether contains something this aspect + /// recognizes and supports. Returns `true` if it can handle the data, + /// `false` otherwise. + /// + abstract static bool ProbeAspect (Stream stream); + + /// + /// Load the aspect and return instance of a class implementing support for it. + /// The parameter can be anything that makes + /// sense for the given aspect (e.g. a file name). + /// + abstract static IAspect LoadAspect (Stream stream, string description); +} diff --git a/tools/apput/src/Common/NativeArchitecture.cs b/tools/apput/src/Common/NativeArchitecture.cs new file mode 100644 index 00000000000..919dc142abc --- /dev/null +++ b/tools/apput/src/Common/NativeArchitecture.cs @@ -0,0 +1,11 @@ +namespace ApplicationUtility; + +public enum NativeArchitecture +{ + Unknown, + + Arm, + Arm64, + X86, + X64, +} \ No newline at end of file diff --git a/tools/apput/src/Native/SharedLibrary.cs b/tools/apput/src/Native/SharedLibrary.cs new file mode 100644 index 00000000000..b807c6f6782 --- /dev/null +++ b/tools/apput/src/Native/SharedLibrary.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; + +using ApplicationUtility; + +public class SharedLibrary : IAspect +{ + public static string AspectName { get; } = "Native shared library"; + + public bool HasAndroidPayload => payloadSize > 0; + public string Name => libraryName; + + readonly ulong payloadOffset; + readonly ulong payloadSize; + readonly string libraryName; + + SharedLibrary (Stream stream, string libraryName) + { + (payloadOffset, payloadSize) = FindAndroidPayload (stream); + this.libraryName = libraryName; + } + + public static IAspect LoadAspect (Stream stream, string description) + { + if (!IsELFSharedLibrary (stream)) { + throw new InvalidOperationException ("Stream is not an ELF shared library"); + } + + return new SharedLibrary (stream, description); + } + + public static bool ProbeAspect (Stream stream) => IsELFSharedLibrary (stream); + + /// + /// If the library has .NET for Android payload section, this + /// method will read the data and write it to the + /// stream. All the data in the output stream will be overwritten. + /// + public void CopyAndroidPayload (Stream dest) + { + Stream payload = OpenAndroidPayload (); + throw new NotImplementedException (); + } + + /// + /// Creates a stream referring to the Android payload data inside + /// the shared library. No data is read, the open stream is returned + /// to the user. Ownership of the stream is transferred to the caller. + /// + public Stream OpenAndroidPayload () + { + if (!HasAndroidPayload) { + throw new InvalidOperationException ("Payload section not found"); + } + + throw new NotImplementedException (); + } + + static bool IsELFSharedLibrary (Stream stream) + { + throw new NotImplementedException (); + } + + (ulong offset, ulong size) FindAndroidPayload (Stream stream) + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs new file mode 100644 index 00000000000..4ce40927a01 --- /dev/null +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -0,0 +1,21 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +public abstract class ApplicationPackage : IAspect +{ + public static string AspectName { get; } = "Application package"; + + public abstract string PackageFormat { get; } + + public static IAspect LoadAspect (Stream stream, string description) + { + throw new NotImplementedException (); + } + + public static bool ProbeAspect (Stream stream) + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/Package/PackageAAB.cs b/tools/apput/src/Package/PackageAAB.cs new file mode 100644 index 00000000000..49a6748b40c --- /dev/null +++ b/tools/apput/src/Package/PackageAAB.cs @@ -0,0 +1,6 @@ +namespace ApplicationUtility; + +class PackageAAB : ApplicationPackage +{ + public override string PackageFormat { get; } = "AAB package"; +} diff --git a/tools/apput/src/Package/PackageAPK.cs b/tools/apput/src/Package/PackageAPK.cs new file mode 100644 index 00000000000..bad2317a666 --- /dev/null +++ b/tools/apput/src/Package/PackageAPK.cs @@ -0,0 +1,6 @@ +namespace ApplicationUtility; + +class PackageAPK : ApplicationPackage +{ + public override string PackageFormat { get; } = "APK package"; +} diff --git a/tools/apput/src/Package/PackageBase.cs b/tools/apput/src/Package/PackageBase.cs new file mode 100644 index 00000000000..d218daf5845 --- /dev/null +++ b/tools/apput/src/Package/PackageBase.cs @@ -0,0 +1,6 @@ +namespace ApplicationUtility; + +class PackageBase : ApplicationPackage +{ + public override string PackageFormat { get; } = "Base application package"; +} diff --git a/tools/apput/src/Program.cs b/tools/apput/src/Program.cs new file mode 100644 index 00000000000..c3e891cbb0a --- /dev/null +++ b/tools/apput/src/Program.cs @@ -0,0 +1,8 @@ +namespace ApplicationUtility; + +class Program +{ + static void Main(string[] args) + { + } +} diff --git a/tools/apput/src/apput.csproj b/tools/apput/src/apput.csproj new file mode 100644 index 00000000000..03f2193cf54 --- /dev/null +++ b/tools/apput/src/apput.csproj @@ -0,0 +1,28 @@ + + + + + Microsoft Corporation + 2025 Microsoft Corporation + 0.0.1 + $(DotNetStableTargetFramework) + false + ../../bin/$(Configuration)/bin/apput + Exe + true + enable + Major + disable + + + + + + + + + + + + + From c2f39d0475e2aa22e953bb2d667f33c3c5fea797 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 4 Jul 2025 17:41:43 +0200 Subject: [PATCH 02/12] Slight IAspect changes + some flesh for the skeleton --- .../ApplicationAssembly.cs | 4 +- .../apput/src/AssemblyStore/AssemblyStore.cs | 4 +- tools/apput/src/Common/IAspect.cs | 12 +- tools/apput/src/Common/Log.cs | 149 ++++++++++++++++++ tools/apput/src/Common/NativeArchitecture.cs | 15 +- tools/apput/src/Detector.cs | 67 ++++++++ tools/apput/src/Native/SharedLibrary.cs | 12 +- tools/apput/src/Package/ApplicationPackage.cs | 99 +++++++++++- tools/apput/src/Package/ApplicationRuntime.cs | 8 + tools/apput/src/Package/PackageAAB.cs | 6 + tools/apput/src/Package/PackageAPK.cs | 6 + tools/apput/src/Package/PackageBase.cs | 6 + tools/apput/src/Program.cs | 4 +- tools/apput/src/apput.csproj | 5 +- 14 files changed, 370 insertions(+), 27 deletions(-) create mode 100644 tools/apput/src/Common/Log.cs create mode 100644 tools/apput/src/Detector.cs create mode 100644 tools/apput/src/Package/ApplicationRuntime.cs diff --git a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs index 37427304ca2..211ccb47dde 100644 --- a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs +++ b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs @@ -13,12 +13,12 @@ public class ApplicationAssembly : IAspect public ulong Size { get; private set; } public bool IgnoreOnLoad { get; private set; } - public static IAspect LoadAspect (Stream stream, string description) + public static IAspect LoadAspect (Stream stream, string? description) { throw new NotImplementedException (); } - public static bool ProbeAspect (Stream stream) + public static bool ProbeAspect (Stream stream, string? description) { throw new NotImplementedException (); } diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index 32c7f96efa4..ba36988088f 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -12,12 +12,12 @@ public class AssemblyStore : IAspect public NativeArchitecture Architecture { get; private set; } = NativeArchitecture.Unknown; public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); - public static IAspect LoadAspect (Stream stream, string description) + public static IAspect LoadAspect (Stream stream, string? description) { throw new NotImplementedException (); } - public static bool ProbeAspect (Stream stream) + public static bool ProbeAspect (Stream stream, string? description) { throw new NotImplementedException (); } diff --git a/tools/apput/src/Common/IAspect.cs b/tools/apput/src/Common/IAspect.cs index c078a5e9fb2..62654673f3f 100644 --- a/tools/apput/src/Common/IAspect.cs +++ b/tools/apput/src/Common/IAspect.cs @@ -1,9 +1,10 @@ +using System; using System.IO; namespace ApplicationUtility; /// -/// Represets an aspect of a .NET for Android application. An aspect can be an +/// Represents an aspect of a .NET for Android application. An aspect can be an /// individual assembly, the whole APK/AAB package, a shared library etc. /// If it exists as a definable, separate entity in the application, that can /// be identified/detected by looking at its format/location it is most @@ -14,19 +15,20 @@ public interface IAspect /// /// Aspect name, for presentation purposes. /// - abstract static string AspectName { get; } + static string AspectName => throw new NotImplementedException (); /// /// Probes whether contains something this aspect /// recognizes and supports. Returns `true` if it can handle the data, - /// `false` otherwise. + /// `false` otherwise. The parameter can be anything that makes + /// sense for the given aspect (e.g. a file name). /// - abstract static bool ProbeAspect (Stream stream); + static bool ProbeAspect (Stream stream, string? description = null) => throw new NotImplementedException (); /// /// Load the aspect and return instance of a class implementing support for it. /// The parameter can be anything that makes /// sense for the given aspect (e.g. a file name). /// - abstract static IAspect LoadAspect (Stream stream, string description); + static IAspect LoadAspect (Stream stream, string? description = null) => throw new NotImplementedException (); } diff --git a/tools/apput/src/Common/Log.cs b/tools/apput/src/Common/Log.cs new file mode 100644 index 00000000000..b9815b1907f --- /dev/null +++ b/tools/apput/src/Common/Log.cs @@ -0,0 +1,149 @@ +using System; + +namespace ApplicationUtility; + +static class Log +{ + public const ConsoleColor ErrorColor = ConsoleColor.Red; + public const ConsoleColor WarningColor = ConsoleColor.Yellow; + public const ConsoleColor InfoColor = ConsoleColor.Green; + public const ConsoleColor DebugColor = ConsoleColor.DarkGray; + + static bool showDebug = false; + + static void WriteStderr (string message) + { + Console.Error.Write (message); + } + + static void WriteStderr (ConsoleColor color, string message) + { + ConsoleColor oldFG = Console.ForegroundColor; + Console.ForegroundColor = color; + WriteStderr (message); + Console.ForegroundColor = oldFG; + } + + static void WriteLineStderr (string message) + { + Console.Error.WriteLine (message); + } + + static void WriteLineStderr (ConsoleColor color, string message) + { + ConsoleColor oldFG = Console.ForegroundColor; + Console.ForegroundColor = color; + WriteLineStderr (message); + Console.ForegroundColor = oldFG; + } + + static void Write (string message) + { + Console.Write (message); + } + + static void Write (ConsoleColor color, string message) + { + ConsoleColor oldFG = Console.ForegroundColor; + Console.ForegroundColor = color; + Write (message); + Console.ForegroundColor = oldFG; + } + + static void WriteLine (string message) + { + Console.WriteLine (message); + } + + static void WriteLine (ConsoleColor color, string message) + { + ConsoleColor oldFG = Console.ForegroundColor; + Console.ForegroundColor = color; + WriteLine (message); + Console.ForegroundColor = oldFG; + } + + public static void SetVerbose (bool verbose) + { + showDebug = verbose; + } + + public static void Error (string message = "") + { + Error (tag: String.Empty, message); + } + + public static void Error (string tag, string message) + { + if (message.Length > 0) { + WriteStderr (ErrorColor, "[E] "); + } + + if (tag.Length > 0) { + WriteStderr (ErrorColor, $"{tag}: "); + } + + WriteLineStderr (message); + } + + public static void Warning (string message = "") + { + Warning (tag: String.Empty, message); + } + + public static void Warning (string tag, string message) + { + if (message.Length > 0) { + WriteStderr (WarningColor, "[W] "); + } + + if (tag.Length > 0) { + WriteStderr (WarningColor, $"{tag}: "); + } + + WriteLineStderr (message); + } + + public static void Info (string message = "") + { + Info (tag: String.Empty, message); + } + + public static void Info (string tag, string message) + { + if (tag.Length > 0) { + Write (InfoColor, $"{tag}: "); + } + + WriteLine (InfoColor,message); + } + + public static void Debug (string message = "") + { + Debug (tag: String.Empty, message); + } + + public static void Debug (string tag, string message) + { + if (!showDebug) { + return; + } + + if (message.Length > 0) { + Write (DebugColor, "[D] "); + } + + if (tag.Length > 0) { + Write (DebugColor, $"{tag}: "); + } + + WriteLine (message); + } + + public static void ExceptionError (string message, Exception ex) + { + Log.Error (message); + Log.Error ("Exception was thrown:"); + Log.Error (ex.ToString ()); + } +} diff --git a/tools/apput/src/Common/NativeArchitecture.cs b/tools/apput/src/Common/NativeArchitecture.cs index 919dc142abc..90a9cac70b0 100644 --- a/tools/apput/src/Common/NativeArchitecture.cs +++ b/tools/apput/src/Common/NativeArchitecture.cs @@ -1,11 +1,14 @@ +using System; + namespace ApplicationUtility; +[Flags] public enum NativeArchitecture { - Unknown, + Unknown = 0x00, - Arm, - Arm64, - X86, - X64, -} \ No newline at end of file + Arm = 0x01, + Arm64 = 0x02, + X86 = 0x04, + X64 = 0x08, +} diff --git a/tools/apput/src/Detector.cs b/tools/apput/src/Detector.cs new file mode 100644 index 00000000000..61495c4bf42 --- /dev/null +++ b/tools/apput/src/Detector.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +namespace ApplicationUtility; + +/// +/// Given path to a file, or a stream, this class tries to +/// detect whether or not the thing is an application aspect +/// we know or we can handle. +/// +class Detector +{ + readonly static List KnownTopLevelAspects = new () { + typeof (ApplicationPackage), + typeof (AssemblyStore), + typeof (ApplicationAssembly), + typeof (SharedLibrary), + }; + + public static IAspect? FindAspect (string path) + { + Log.Debug ($"Looking for aspect matching '{path}'"); + if (!File.Exists (path)) { + return null; + } + + using Stream fs = File.OpenRead (path); + return TryFindAspect (fs, path); + } + + public static IAspect? FindAspect (Stream stream, string? description = null) + { + Log.Debug ($"Looking for aspect supporting a stream ('{description}')"); + return TryFindAspect (stream, description); + } + + static IAspect? TryFindAspect (Stream stream, string? description) + { + var args = new object?[] { stream, description }; + var flags = BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static; + + foreach (Type aspect in KnownTopLevelAspects) { + Log.Debug ($"Probing aspect: {aspect}"); + + object? result = aspect.InvokeMember ( + "ProbeAspect", flags, null, null, args + ); + + if (result == null || (result is bool success && !success)) { + continue; + } + + Log.Debug ($"Loading aspect: {aspect}"); + result = aspect.InvokeMember ( + "LoadAspect", flags, null, null, args + ); + if (result != null) { + return (IAspect)result; + } + } + + return null; + } + +} diff --git a/tools/apput/src/Native/SharedLibrary.cs b/tools/apput/src/Native/SharedLibrary.cs index b807c6f6782..ce5acd17515 100644 --- a/tools/apput/src/Native/SharedLibrary.cs +++ b/tools/apput/src/Native/SharedLibrary.cs @@ -20,16 +20,20 @@ public class SharedLibrary : IAspect this.libraryName = libraryName; } - public static IAspect LoadAspect (Stream stream, string description) + public static IAspect LoadAspect (Stream stream, string? description) { - if (!IsELFSharedLibrary (stream)) { + if (String.IsNullOrEmpty (description)) { + throw new ArgumentException ("Must be a shared library name", nameof (description)); + } + + if (!IsELFSharedLibrary (stream, description)) { throw new InvalidOperationException ("Stream is not an ELF shared library"); } return new SharedLibrary (stream, description); } - public static bool ProbeAspect (Stream stream) => IsELFSharedLibrary (stream); + public static bool ProbeAspect (Stream stream, string? description) => IsELFSharedLibrary (stream, description); /// /// If the library has .NET for Android payload section, this @@ -56,7 +60,7 @@ public Stream OpenAndroidPayload () throw new NotImplementedException (); } - static bool IsELFSharedLibrary (Stream stream) + static bool IsELFSharedLibrary (Stream stream, string? description) { throw new NotImplementedException (); } diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index 4ce40927a01..eaebcae440d 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -1,21 +1,112 @@ using System; +using System.Collections.Generic; using System.IO; +using System.IO.Compression; +using System.Linq; namespace ApplicationUtility; public abstract class ApplicationPackage : IAspect { + readonly static HashSet KnownApkEntries = new (StringComparer.Ordinal) { + "AndroidManifest.xml", + "classes.dex", + }; + + readonly static HashSet KnownAabEntries = new (StringComparer.Ordinal) { + "BundleConfig.pb", + "base/manifest/AndroidManifest.xml", + "base/dex/classes.dex", + }; + + readonly static HashSet KnownBaseEntries = new (StringComparer.Ordinal) { + "manifest/AndroidManifest.xml", + "dex/classes.dex", + }; + public static string AspectName { get; } = "Application package"; public abstract string PackageFormat { get; } - public static IAspect LoadAspect (Stream stream, string description) + protected ZipArchive Zip { get; } + public string? Description { get; } + + public bool Signed { get; protected set; } + public ApplicationRuntime Runtime { get; protected set; } = ApplicationRuntime.Unknown; + public string PackageName { get; protected set; } = ""; + public List? AssemblyStores { get; protected set; } + public NativeArchitecture Architectures { get; protected set; } + + protected ApplicationPackage (ZipArchive zip, string? description) + { + Zip = zip; + Description = description; + } + + public static IAspect LoadAspect (Stream stream, string? description) + { + Log.Debug ($"ApplicationPackage: opening stream ('{description}') as a ZIP archive"); + ZipArchive? zip = TryOpenAsZip (stream); + if (zip == null) { + throw new InvalidOperationException ("Stream is not a ZIP archive. Call ProbeAspect first."); + } + + ApplicationPackage ret; + if (IsAPK (zip)) { + ret = new PackageAPK (zip, description); + } else if (IsAAB (zip)) { + ret = new PackageAAB (zip, description); + } else if (IsBase (zip)) { + ret = new PackageBase (zip, description); + } else { + throw new InvalidOperationException ("Stream is not a supported Android ZIP package. Call ProbeAspect first."); + } + + Log.Debug ($"ApplicationPackage: stream ('{description}') is: {ret.PackageFormat}"); + return ret; + } + + public static bool ProbeAspect (Stream stream, string? description) + { + Log.Debug ($"ApplicationPackage: checking if stream ('{description}') is a ZIP archive"); + using ZipArchive? zip = TryOpenAsZip (stream); + if (zip == null) { + return false; + } + + Log.Debug ($"ApplicationPackage: checking if stream ('{description}') is a supported Android ZIP package"); + // OK, it's a ZIP. Find out if it's what we support + string? kind = null; + if (IsAPK (zip)) { + kind = "APK"; + } else if (IsAAB (zip)) { + kind = "AAB"; + } else if (IsBase (zip)) { + kind = "Base"; + } else { + return false; + } + + Log.Debug ($"ApplicationPackage: archive is {kind}"); + return true; + } + + static bool IsAPK (ZipArchive zip) => HasEntries (zip, KnownApkEntries); + static bool IsAAB (ZipArchive zip) => HasEntries (zip, KnownAabEntries); + static bool IsBase (ZipArchive zip) => HasEntries (zip, KnownBaseEntries); + + static bool HasEntries (ZipArchive zip, HashSet knownEntries) { - throw new NotImplementedException (); + return zip.Entries.Where ((ZipArchiveEntry entry) => knownEntries.Contains (entry.FullName)).Any (); } - public static bool ProbeAspect (Stream stream) + static ZipArchive? TryOpenAsZip (Stream stream) { - throw new NotImplementedException (); + stream.Seek (0, SeekOrigin.Begin); + try { + return new ZipArchive (stream, ZipArchiveMode.Read, leaveOpen: true); + } catch (InvalidDataException) { + return null; + } } } diff --git a/tools/apput/src/Package/ApplicationRuntime.cs b/tools/apput/src/Package/ApplicationRuntime.cs new file mode 100644 index 00000000000..0b0922a737c --- /dev/null +++ b/tools/apput/src/Package/ApplicationRuntime.cs @@ -0,0 +1,8 @@ +namespace ApplicationUtility; + +public enum ApplicationRuntime +{ + Unknown, + MonoVM, + CoreCLR, +} diff --git a/tools/apput/src/Package/PackageAAB.cs b/tools/apput/src/Package/PackageAAB.cs index 49a6748b40c..cbccd69fdcf 100644 --- a/tools/apput/src/Package/PackageAAB.cs +++ b/tools/apput/src/Package/PackageAAB.cs @@ -1,6 +1,12 @@ +using System.IO.Compression; + namespace ApplicationUtility; class PackageAAB : ApplicationPackage { public override string PackageFormat { get; } = "AAB package"; + + public PackageAAB (ZipArchive zip, string? description) + : base (zip, description) + {} } diff --git a/tools/apput/src/Package/PackageAPK.cs b/tools/apput/src/Package/PackageAPK.cs index bad2317a666..e2c3b939d58 100644 --- a/tools/apput/src/Package/PackageAPK.cs +++ b/tools/apput/src/Package/PackageAPK.cs @@ -1,6 +1,12 @@ +using System.IO.Compression; + namespace ApplicationUtility; class PackageAPK : ApplicationPackage { public override string PackageFormat { get; } = "APK package"; + + public PackageAPK (ZipArchive zip, string? description) + : base (zip, description) + {} } diff --git a/tools/apput/src/Package/PackageBase.cs b/tools/apput/src/Package/PackageBase.cs index d218daf5845..b5977b520c4 100644 --- a/tools/apput/src/Package/PackageBase.cs +++ b/tools/apput/src/Package/PackageBase.cs @@ -1,6 +1,12 @@ +using System.IO.Compression; + namespace ApplicationUtility; class PackageBase : ApplicationPackage { public override string PackageFormat { get; } = "Base application package"; + + public PackageBase (ZipArchive zip, string? description) + : base (zip, description) + {} } diff --git a/tools/apput/src/Program.cs b/tools/apput/src/Program.cs index c3e891cbb0a..f8d37657343 100644 --- a/tools/apput/src/Program.cs +++ b/tools/apput/src/Program.cs @@ -2,7 +2,9 @@ namespace ApplicationUtility; class Program { - static void Main(string[] args) + static void Main (string[] args) { + Log.SetVerbose (true); + IAspect? aspect = Detector.FindAspect (args[0]); } } diff --git a/tools/apput/src/apput.csproj b/tools/apput/src/apput.csproj index 03f2193cf54..a052461f525 100644 --- a/tools/apput/src/apput.csproj +++ b/tools/apput/src/apput.csproj @@ -1,11 +1,11 @@ - + Microsoft Corporation 2025 Microsoft Corporation 0.0.1 - $(DotNetStableTargetFramework) + $(DotNetStableTargetFramework) false ../../bin/$(Configuration)/bin/apput Exe @@ -19,7 +19,6 @@ - From 27c11f2558267e096b0cf61d2154568188585c45 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 7 Jul 2025 15:02:20 +0200 Subject: [PATCH 03/12] A few steps more --- .../apput/src/AssemblyStore/AssemblyStore.cs | 4 +- tools/apput/src/Common/Log.cs | 11 ++ tools/apput/src/Package/ApplicationPackage.cs | 176 +++++++++++++++++- tools/apput/src/Package/ApplicationRuntime.cs | 1 + tools/apput/src/Package/PackageAAB.cs | 2 + tools/apput/src/Package/PackageAPK.cs | 2 + tools/apput/src/Package/PackageBase.cs | 2 + tools/apput/src/apput.csproj | 8 + 8 files changed, 199 insertions(+), 7 deletions(-) diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index ba36988088f..525440cfd7d 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; +using Xamarin.Android.Tools; + namespace ApplicationUtility; public class AssemblyStore : IAspect @@ -9,7 +11,7 @@ public class AssemblyStore : IAspect public static string AspectName { get; } = "Assembly Store"; public IDictionary Assemblies { get; private set; } = null!; - public NativeArchitecture Architecture { get; private set; } = NativeArchitecture.Unknown; + public AndroidTargetArch Architecture { get; private set; } = AndroidTargetArch.None; public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); public static IAspect LoadAspect (Stream stream, string? description) diff --git a/tools/apput/src/Common/Log.cs b/tools/apput/src/Common/Log.cs index b9815b1907f..6ffef4d784e 100644 --- a/tools/apput/src/Common/Log.cs +++ b/tools/apput/src/Common/Log.cs @@ -123,6 +123,7 @@ public static void Debug (string message = "") Debug (tag: String.Empty, message); } + // TODO: debug should go to file if verbose output isn't enabled public static void Debug (string tag, string message) { if (!showDebug) { @@ -140,6 +141,16 @@ public static void Debug (string tag, string message) WriteLine (message); } + public static void Debug (string message, Exception ex) + { + if (!showDebug) { + return; + } + + Debug (tag: String.Empty, message); + Debug (tag: String.Empty, ex.ToString ()); + } + public static void ExceptionError (string message, Exception ex) { Log.Error (message); diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index eaebcae440d..ef95f1edc60 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -4,6 +4,9 @@ using System.IO.Compression; using System.Linq; +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; + namespace ApplicationUtility; public abstract class ApplicationPackage : IAspect @@ -24,18 +27,28 @@ public abstract class ApplicationPackage : IAspect "dex/classes.dex", }; + readonly static HashSet KnownSignatureEntries = new (StringComparer.Ordinal) { + "META-INF/BNDLTOOL.RSA", + "META-INF/ANDROIDD.RSA", + }; + public static string AspectName { get; } = "Application package"; public abstract string PackageFormat { get; } + protected abstract string NativeLibDirBase { get; } + protected abstract string AndroidManifestPath { get; } protected ZipArchive Zip { get; } public string? Description { get; } public bool Signed { get; protected set; } + public bool ValidAndroidPackage { get; protected set; } + public bool Debuggable { get; protected set; } public ApplicationRuntime Runtime { get; protected set; } = ApplicationRuntime.Unknown; public string PackageName { get; protected set; } = ""; + public string MainActivity { get; protected set; } = ""; public List? AssemblyStores { get; protected set; } - public NativeArchitecture Architectures { get; protected set; } + public List Architectures { get; protected set; } = new (); protected ApplicationPackage (ZipArchive zip, string? description) { @@ -61,11 +74,147 @@ public static IAspect LoadAspect (Stream stream, string? description) } else { throw new InvalidOperationException ("Stream is not a supported Android ZIP package. Call ProbeAspect first."); } - Log.Debug ($"ApplicationPackage: stream ('{description}') is: {ret.PackageFormat}"); + + // TODO: for all of the below, add support for detection of older XA apps (just to warn that this version doesn't support + // and that people should use older tools) + ret.TryDetectArchitectures (); // This must be called first, some further steps depend on it + ret.TryDetectRuntime (); + ret.TryDetectWhetherIsSigned (); + ret.TryLoadAssemblyStores (); + ret.TryLoadAndroidManifest (); + return ret; } + void TryDetectArchitectures () + { + foreach (AndroidTargetArch arch in Enum.GetValues ()) { + if (!MonoAndroidHelper.SupportedTargetArchitectures.Contains (arch)) { + continue; + } + + // We can't simply test for presence of the libDir below, because it's possible + // that a separate entry for the "directory" (they are only a naming convention + // in the ZIP archive, not a separate entity) won't exist. Instead, we look for + // any entry starting with the path. + if (!HasEntryStartingWith (Zip, GetNativeLibDir (arch))) { + continue; + } + Architectures.Add (arch); + Log.Debug ($"Detected architecture: {arch}"); + } + } + + void TryDetectRuntime () + { + ApplicationRuntime runtime = ApplicationRuntime.Unknown; + string runtimePath; + foreach (AndroidTargetArch arch in Architectures) { + runtimePath = GetNativeLibFile (arch, "libcoreclr.so"); + if (HasEntry (Zip, runtimePath)) { + runtime = ApplicationRuntime.CoreCLR; + break; + } + + runtimePath = GetNativeLibFile (arch, "libmonosgen-2.0.so"); + if (HasEntry (Zip, runtimePath)) { + runtime = ApplicationRuntime.MonoVM; + break; + } + } + + if (runtime != ApplicationRuntime.Unknown || Architectures.Count == 0) { + Log.Debug ($"Detected runtime: {runtime}"); + return; + } + + runtimePath = GetNativeLibFile (Architectures[0], "libmonodroid.so"); + if (!HasEntry (Zip, runtimePath)) { + return; + } + + // TODO: it might be statically linked CoreCLR runtime. Need to check for presence of + // some public symbols to verify that. + } + + void TryDetectWhetherIsSigned () + { + Signed = HasAnyEntries (Zip, KnownSignatureEntries); + Log.Debug ($"Signature detected: {Signed}"); + } + + void TryLoadAssemblyStores () + { + foreach (AndroidTargetArch arch in Architectures) { + string storePath = GetNativeLibFile (arch, $"libassemblies.{MonoAndroidHelper.ArchToAbi (arch)}.blob.so"); + Log.Debug ($"Trying assembly store: {storePath}"); + if (!HasEntry (Zip, storePath)) { + Log.Debug ($"Assembly store '{storePath}' not found"); + continue; + } + + Log.Debug ($"Found assembly store entry for architecture {arch}"); + AssemblyStore? store = TryLoadAssemblyStore (storePath); + if (store == null) { + continue; + } + } + } + + AssemblyStore? TryLoadAssemblyStore (string storePath) + { + // AssemblyStore class owns the stream, don't dispose it here + Stream? storeStream = TryGetEntryStream (storePath); + if (storeStream == null) { + return null; + } + + try { + if (!AssemblyStore.ProbeAspect (storeStream, storePath)) { + Log.Debug ($"Assembly store '{storePath}' is not in a supported format"); + return null; + } + + return (AssemblyStore)AssemblyStore.LoadAspect (storeStream, storePath); + } catch (Exception ex) { + Log.Debug ($"Failed to load assembly store '{storePath}'", ex); + return null; + } + } + + void TryLoadAndroidManifest () + { + ValidAndroidPackage = HasEntry (Zip, AndroidManifestPath); + if (!ValidAndroidPackage) { + Log.Debug ($"Package is missing manifest entry '{AndroidManifestPath}'"); + return; + } + + Log.Debug ($"Found Android manifest '{AndroidManifestPath}'"); + using Stream? manifestStream = TryGetEntryStream (AndroidManifestPath); + // TODO: parse + } + + string GetNativeLibDir (AndroidTargetArch arch) => $"{NativeLibDirBase}/{MonoAndroidHelper.ArchToAbi (arch)}/"; + string GetNativeLibFile (AndroidTargetArch arch, string fileName) => $"{GetNativeLibDir (arch)}{fileName}"; + + Stream? TryGetEntryStream (string path) + { + try { + ZipArchiveEntry? entry = Zip.GetEntry (path); + if (entry == null) { + Log.Debug ($"ZIP entry '{path}' could not be loaded."); + return null; + } + + return entry.Open (); + } catch (Exception ex) { + Log.Debug ($"Failed to load entry '{path}' from the archive.", ex); + return null; + } + } + public static bool ProbeAspect (Stream stream, string? description) { Log.Debug ($"ApplicationPackage: checking if stream ('{description}') is a ZIP archive"); @@ -91,15 +240,30 @@ public static bool ProbeAspect (Stream stream, string? description) return true; } - static bool IsAPK (ZipArchive zip) => HasEntries (zip, KnownApkEntries); - static bool IsAAB (ZipArchive zip) => HasEntries (zip, KnownAabEntries); - static bool IsBase (ZipArchive zip) => HasEntries (zip, KnownBaseEntries); + static bool IsAPK (ZipArchive zip) => HasAllEntries (zip, KnownApkEntries); + static bool IsAAB (ZipArchive zip) => HasAllEntries (zip, KnownAabEntries); + static bool IsBase (ZipArchive zip) => HasAllEntries (zip, KnownBaseEntries); - static bool HasEntries (ZipArchive zip, HashSet knownEntries) + static bool HasAnyEntries (ZipArchive zip, HashSet knownEntries) { return zip.Entries.Where ((ZipArchiveEntry entry) => knownEntries.Contains (entry.FullName)).Any (); } + static bool HasAllEntries (ZipArchive zip, HashSet knownEntries) + { + return zip.Entries.Where ((ZipArchiveEntry entry) => knownEntries.Contains (entry.FullName)).Count () == knownEntries.Count; + } + + static bool HasEntry (ZipArchive zip, string path) + { + return zip.Entries.Where ((ZipArchiveEntry entry) => entry.FullName == path).Any (); + } + + static bool HasEntryStartingWith (ZipArchive zip, string path) + { + return zip.Entries.Where ((ZipArchiveEntry entry) => entry.FullName.StartsWith (path, StringComparison.Ordinal)).Any (); + } + static ZipArchive? TryOpenAsZip (Stream stream) { stream.Seek (0, SeekOrigin.Begin); diff --git a/tools/apput/src/Package/ApplicationRuntime.cs b/tools/apput/src/Package/ApplicationRuntime.cs index 0b0922a737c..9efa9ff598c 100644 --- a/tools/apput/src/Package/ApplicationRuntime.cs +++ b/tools/apput/src/Package/ApplicationRuntime.cs @@ -5,4 +5,5 @@ public enum ApplicationRuntime Unknown, MonoVM, CoreCLR, + StaticCoreCLR, } diff --git a/tools/apput/src/Package/PackageAAB.cs b/tools/apput/src/Package/PackageAAB.cs index cbccd69fdcf..6cc96068e60 100644 --- a/tools/apput/src/Package/PackageAAB.cs +++ b/tools/apput/src/Package/PackageAAB.cs @@ -5,6 +5,8 @@ namespace ApplicationUtility; class PackageAAB : ApplicationPackage { public override string PackageFormat { get; } = "AAB package"; + protected override string NativeLibDirBase => "base/lib"; + protected override string AndroidManifestPath => "base/manifest/AndroidManifest.xml"; public PackageAAB (ZipArchive zip, string? description) : base (zip, description) diff --git a/tools/apput/src/Package/PackageAPK.cs b/tools/apput/src/Package/PackageAPK.cs index e2c3b939d58..fa8120fc0fd 100644 --- a/tools/apput/src/Package/PackageAPK.cs +++ b/tools/apput/src/Package/PackageAPK.cs @@ -5,6 +5,8 @@ namespace ApplicationUtility; class PackageAPK : ApplicationPackage { public override string PackageFormat { get; } = "APK package"; + protected override string NativeLibDirBase => "lib"; + protected override string AndroidManifestPath => "AndroidManifest.xml"; public PackageAPK (ZipArchive zip, string? description) : base (zip, description) diff --git a/tools/apput/src/Package/PackageBase.cs b/tools/apput/src/Package/PackageBase.cs index b5977b520c4..738ec6fd858 100644 --- a/tools/apput/src/Package/PackageBase.cs +++ b/tools/apput/src/Package/PackageBase.cs @@ -5,6 +5,8 @@ namespace ApplicationUtility; class PackageBase : ApplicationPackage { public override string PackageFormat { get; } = "Base application package"; + protected override string NativeLibDirBase => "lib"; + protected override string AndroidManifestPath => "manifest/AndroidManifest.xml"; public PackageBase (ZipArchive zip, string? description) : base (zip, description) diff --git a/tools/apput/src/apput.csproj b/tools/apput/src/apput.csproj index a052461f525..e70dffcd648 100644 --- a/tools/apput/src/apput.csproj +++ b/tools/apput/src/apput.csproj @@ -24,4 +24,12 @@ + + + + + + + + From 690762d90c9317377c62eafe9cab31a2b0fda0a9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 7 Jul 2025 19:04:57 +0200 Subject: [PATCH 04/12] Assembly stores progress --- .../apput/src/AssemblyStore/AssemblyStore.cs | 41 ++++++++++++++++++- .../src/AssemblyStore/AssemblyStoreAbi.cs | 10 +++++ .../src/AssemblyStore/AssemblyStoreHeader.cs | 20 +++++++++ .../src/AssemblyStore/AssemblyStoreVersion.cs | 15 +++++++ tools/apput/src/AssemblyStore/Format_V3.cs | 5 +++ tools/apput/src/Common/Utilities.cs | 34 +++++++++++++++ tools/apput/src/Package/ApplicationPackage.cs | 14 +++++-- tools/apput/src/Program.cs | 25 ++++++++--- 8 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs create mode 100644 tools/apput/src/AssemblyStore/Format_V3.cs create mode 100644 tools/apput/src/Common/Utilities.cs diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index 525440cfd7d..563214dd9b5 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using Xamarin.Android.Tools; @@ -8,9 +9,12 @@ namespace ApplicationUtility; public class AssemblyStore : IAspect { + const int MinimumStoreSize = 8; + const uint MagicNumber = 0x41424158; // 'XABA', little-endian + public static string AspectName { get; } = "Assembly Store"; - public IDictionary Assemblies { get; private set; } = null!; + public IDictionary Assemblies { get; private set; } = new Dictionary (StringComparer.Ordinal); public AndroidTargetArch Architecture { get; private set; } = AndroidTargetArch.None; public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); @@ -21,6 +25,39 @@ public static IAspect LoadAspect (Stream stream, string? description) public static bool ProbeAspect (Stream stream, string? description) { - throw new NotImplementedException (); + // TODO: check if it's an ELF .so and extract the payload, if necessary + + // All assembly store files are at least 8 bytes long - space taken up by + // the magic number + store version. + if (stream.Length < MinimumStoreSize) { + Log.Debug ($"AssemblyStore: stream ('{description}') isn't long enough. Need at least {MinimumStoreSize} bytes"); + return false; + } + + stream.Seek (0, SeekOrigin.Begin); + using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); + uint magic = reader.ReadUInt32 (); + if (magic != MagicNumber) { + Log.Debug ($"AssemblyStore: stream ('{description}') doesn't have the correct signature."); + return false; + } + + uint version = reader.ReadUInt32 (); + + // We currently support version 3. Main store version is kept in the lower 16 bits of the version word + uint mainVersion = version & 0xFFFF; + switch (mainVersion) { + case 3: + return ValidateFormatVersion3 (stream, description); + + default: + Log.Debug ($"AssemblyStore: unsupported store version: {mainVersion}"); + return false; + } + } + + static bool ValidateFormatVersion3 (Stream stream, string? description) + { + return true; } } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs new file mode 100644 index 00000000000..e5a0546034d --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs @@ -0,0 +1,10 @@ +namespace ApplicationUtility; + +// Values correspond to those in `xamarin-app.hh` +enum AssemblyStoreABI +{ + Arm64 = 0x00010000, + Arm = 0x00020000, + X86 = 0x00030000, + X64 = 0x00040000, +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs new file mode 100644 index 00000000000..566c95ce46f --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs @@ -0,0 +1,20 @@ +namespace ApplicationUtility; + +/// +/// Represents a high-level description of the store hader. It means that this class +/// does **not** correspond to a physical format of the header in the store file. Instead, +/// it contains all the information gathered from the physical file, in a forward compatible +/// way. Forward compatibility means that all public the properties are virtual and nullable, +/// since it's possible that some of them will not be present in the future revisions of the +/// on-disk structure. No public property shall be removed, but any and all of them may be +/// `null` for any given version of the assembly store format. The only exception to this rule +/// is the `Version` property, since it is expected to be present in one way or another in all +/// the future format revisions. +/// +class AssemblyStoreHeader +{ + public AssemblyStoreVersion Version { get; protected set; } + public uint? EntryCount { get; protected set; } + public uint? IndexEntryCount { get; protected set; } + public uint? IndexSize { get; protected set; } +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs new file mode 100644 index 00000000000..38083a9107c --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs @@ -0,0 +1,15 @@ +namespace ApplicationUtility; + +class AssemblyStoreVersion +{ + public uint MainVersion { get; } + public AssemblyStoreABI ABI { get; } + public bool Is64Bit { get; } + + internal AssemblyStoreVersion (uint mainVersion, AssemblyStoreABI abi, bool is64Bit) + { + MainVersion = mainVersion; + ABI = abi; + Is64Bit = is64Bit; + } +} diff --git a/tools/apput/src/AssemblyStore/Format_V3.cs b/tools/apput/src/AssemblyStore/Format_V3.cs new file mode 100644 index 00000000000..871a9482109 --- /dev/null +++ b/tools/apput/src/AssemblyStore/Format_V3.cs @@ -0,0 +1,5 @@ +namespace ApplicationUtility; + +class Format_V3 +{ +} diff --git a/tools/apput/src/Common/Utilities.cs b/tools/apput/src/Common/Utilities.cs new file mode 100644 index 00000000000..fb2a9f35bc4 --- /dev/null +++ b/tools/apput/src/Common/Utilities.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +class Utilities +{ + public static void DeleteFile (string path, bool quiet = true) + { + try { + File.Delete (path); + } catch (Exception ex) { + Log.Debug ($"Failed to delete file '{path}'.", ex); + if (!quiet) { + throw; + } + } + } + + public static void CloseAndDeleteFile (FileStream stream, bool quiet = true) + { + string path = stream.Name; + try { + stream.Close (); + } catch (Exception ex) { + Log.Debug ($"Failed to close file stream.", ex); + if (!quiet) { + throw; + } + } + + DeleteFile (path); + } +} diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index ef95f1edc60..9aa4c9ccf1a 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -165,7 +165,7 @@ void TryLoadAssemblyStores () AssemblyStore? TryLoadAssemblyStore (string storePath) { // AssemblyStore class owns the stream, don't dispose it here - Stream? storeStream = TryGetEntryStream (storePath); + FileStream? storeStream = TryGetEntryStream (storePath); if (storeStream == null) { return null; } @@ -173,6 +173,7 @@ void TryLoadAssemblyStores () try { if (!AssemblyStore.ProbeAspect (storeStream, storePath)) { Log.Debug ($"Assembly store '{storePath}' is not in a supported format"); + storeStream.Close (); return null; } @@ -199,7 +200,7 @@ void TryLoadAndroidManifest () string GetNativeLibDir (AndroidTargetArch arch) => $"{NativeLibDirBase}/{MonoAndroidHelper.ArchToAbi (arch)}/"; string GetNativeLibFile (AndroidTargetArch arch, string fileName) => $"{GetNativeLibDir (arch)}{fileName}"; - Stream? TryGetEntryStream (string path) + FileStream? TryGetEntryStream (string path) { try { ZipArchiveEntry? entry = Zip.GetEntry (path); @@ -208,9 +209,16 @@ void TryLoadAndroidManifest () return null; } - return entry.Open (); + string tempFile = Path.GetTempFileName (); + TempFileManager.RegisterFile (tempFile); + + Log.Debug ($"Extracting entry '{path}' to '{tempFile}'"); + entry.ExtractToFile (tempFile, overwrite: true); + return File.OpenRead (tempFile); } catch (Exception ex) { Log.Debug ($"Failed to load entry '{path}' from the archive.", ex); + + // TODO: remove temp file (using a helper method, which doesn't exist yet) return null; } } diff --git a/tools/apput/src/Program.cs b/tools/apput/src/Program.cs index f8d37657343..de0b10192d1 100644 --- a/tools/apput/src/Program.cs +++ b/tools/apput/src/Program.cs @@ -1,10 +1,25 @@ +using System; + namespace ApplicationUtility; class Program { - static void Main (string[] args) - { - Log.SetVerbose (true); - IAspect? aspect = Detector.FindAspect (args[0]); - } + static int Main (string[] args) + { + Log.SetVerbose (true); + try { + return Run (args); + } catch (Exception ex) { + Log.ExceptionError ("Unhandled exception", ex); + return 1; + } finally { + TempFileManager.Cleanup (); + } + } + + static int Run (string[] args) + { + IAspect? aspect = Detector.FindAspect (args[0]); + return 0; + } } From 3d886b358df82a09c40dda85399b91b99e21d79f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 8 Jul 2025 09:18:34 +0200 Subject: [PATCH 05/12] Oops, forgot to add this one, doh --- tools/apput/src/Common/TempFileManager.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tools/apput/src/Common/TempFileManager.cs diff --git a/tools/apput/src/Common/TempFileManager.cs b/tools/apput/src/Common/TempFileManager.cs new file mode 100644 index 00000000000..ef75d92f734 --- /dev/null +++ b/tools/apput/src/Common/TempFileManager.cs @@ -0,0 +1,14 @@ +namespace ApplicationUtility; + +class TempFileManager +{ + public static void RegisterFile (string path) + { + // TODO: implement + } + + public static void Cleanup () + { + // TODO: implement + } +} From d83ec12043d42799944d92d03ef1dd8d99810a12 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 8 Jul 2025 16:55:15 +0200 Subject: [PATCH 06/12] Progressing with assembly stores + shared libraries --- .../ApplicationAssembly.cs | 4 +- .../apput/src/AssemblyStore/AssemblyStore.cs | 60 +++++-- .../AssemblyStore/AssemblyStoreAspectState.cs | 13 ++ .../src/AssemblyStore/AssemblyStoreIndex.cs | 5 + tools/apput/src/AssemblyStore/FormatBase.cs | 17 ++ tools/apput/src/AssemblyStore/Format_V2.cs | 16 ++ tools/apput/src/AssemblyStore/Format_V3.cs | 14 +- tools/apput/src/Common/BasicAspectState.cs | 11 ++ tools/apput/src/Common/IAspect.cs | 4 +- tools/apput/src/Common/IAspectState.cs | 15 ++ tools/apput/src/Detector.cs | 8 +- tools/apput/src/Native/SharedLibrary.cs | 161 ++++++++++++++++-- .../src/Native/SharedLibraryPayloadStream.cs | 69 ++++++++ tools/apput/src/Package/ApplicationPackage.cs | 16 +- 14 files changed, 370 insertions(+), 43 deletions(-) create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs create mode 100644 tools/apput/src/AssemblyStore/FormatBase.cs create mode 100644 tools/apput/src/AssemblyStore/Format_V2.cs create mode 100644 tools/apput/src/Common/BasicAspectState.cs create mode 100644 tools/apput/src/Common/IAspectState.cs create mode 100644 tools/apput/src/Native/SharedLibraryPayloadStream.cs diff --git a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs index 211ccb47dde..48f4416a0b6 100644 --- a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs +++ b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs @@ -13,12 +13,12 @@ public class ApplicationAssembly : IAspect public ulong Size { get; private set; } public bool IgnoreOnLoad { get; private set; } - public static IAspect LoadAspect (Stream stream, string? description) + public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) { throw new NotImplementedException (); } - public static bool ProbeAspect (Stream stream, string? description) + public static IAspectState ProbeAspect (Stream stream, string? description) { throw new NotImplementedException (); } diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index 563214dd9b5..916136e6688 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -18,46 +18,78 @@ public class AssemblyStore : IAspect public AndroidTargetArch Architecture { get; private set; } = AndroidTargetArch.None; public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); - public static IAspect LoadAspect (Stream stream, string? description) + public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) { + var storeState = state as AssemblyStoreAspectState; throw new NotImplementedException (); } - public static bool ProbeAspect (Stream stream, string? description) + public static IAspectState ProbeAspect (Stream stream, string? description) { - // TODO: check if it's an ELF .so and extract the payload, if necessary + Stream? storeStream = null; + try { + IAspectState state = SharedLibrary.ProbeAspect (stream, description); + if (!state.Success) { + return DoProbeAspect (stream, description); + } + + var library = (SharedLibrary)SharedLibrary.LoadAspect (stream, state, description); + if (!library.HasAndroidPayload) { + Log.Debug ($"AssemblyStore: stream ('{description}') is an ELF shared library, without payload"); + return new BasicAspectState (false); + } + Log.Debug ($"AssemblyStore: stream ('{description}') is an ELF shared library with .NET for Android payload section"); + storeStream = library.OpenAndroidPayload (); + return DoProbeAspect (storeStream, description); + } finally { + storeStream?.Dispose (); + } + } + + // We return `BasicAspectState` instance for all failures, since there's no extra information we can + // pass on. + static IAspectState DoProbeAspect (Stream storeStream, string? description) + { // All assembly store files are at least 8 bytes long - space taken up by // the magic number + store version. - if (stream.Length < MinimumStoreSize) { + if (storeStream.Length < MinimumStoreSize) { Log.Debug ($"AssemblyStore: stream ('{description}') isn't long enough. Need at least {MinimumStoreSize} bytes"); - return false; + return new BasicAspectState (false); } - stream.Seek (0, SeekOrigin.Begin); - using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); + storeStream.Seek (0, SeekOrigin.Begin); + using var reader = new BinaryReader (storeStream, Encoding.UTF8, leaveOpen: true); uint magic = reader.ReadUInt32 (); if (magic != MagicNumber) { Log.Debug ($"AssemblyStore: stream ('{description}') doesn't have the correct signature."); - return false; + return new BasicAspectState (false); } uint version = reader.ReadUInt32 (); // We currently support version 3. Main store version is kept in the lower 16 bits of the version word uint mainVersion = version & 0xFFFF; + FormatBase? validator = null; + switch (mainVersion) { + case 2: + validator = new Format_V2 (storeStream, description); + break; + case 3: - return ValidateFormatVersion3 (stream, description); + validator = new Format_V3 (storeStream, description); + break; default: Log.Debug ($"AssemblyStore: unsupported store version: {mainVersion}"); - return false; + return new BasicAspectState (false); } - } - static bool ValidateFormatVersion3 (Stream stream, string? description) - { - return true; + if (validator == null) { + throw new InvalidOperationException ("Internal error: validator should never be null here"); + } + + return validator.Validate (); } } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs new file mode 100644 index 00000000000..5f4202ab872 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace ApplicationUtility; + +class AssemblyStoreAspectState : BasicAspectState +{ + public AssemblyStoreHeader Header { get; } + public AssemblyStoreIndex Index { get; } + + public AssemblyStoreAspectState (bool success) + : base (success) + {} +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs b/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs new file mode 100644 index 00000000000..83ae5195d5e --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs @@ -0,0 +1,5 @@ +using ApplicationUtility; + +class AssemblyStoreIndex +{ +} diff --git a/tools/apput/src/AssemblyStore/FormatBase.cs b/tools/apput/src/AssemblyStore/FormatBase.cs new file mode 100644 index 00000000000..d337f03e2fe --- /dev/null +++ b/tools/apput/src/AssemblyStore/FormatBase.cs @@ -0,0 +1,17 @@ +using System.IO; + +namespace ApplicationUtility; + +abstract class FormatBase +{ + protected Stream StoreStream { get; } + protected string? Description { get; } + + protected FormatBase (Stream storeStream, string? description) + { + this.StoreStream = storeStream; + this.Description = description; + } + + public abstract IAspectState Validate (); +} diff --git a/tools/apput/src/AssemblyStore/Format_V2.cs b/tools/apput/src/AssemblyStore/Format_V2.cs new file mode 100644 index 00000000000..cceeeb72347 --- /dev/null +++ b/tools/apput/src/AssemblyStore/Format_V2.cs @@ -0,0 +1,16 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +class Format_V2 : FormatBase +{ + public Format_V2 (Stream storeStream, string? description) + : base (storeStream, description) + {} + + public override IAspectState Validate () + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/AssemblyStore/Format_V3.cs b/tools/apput/src/AssemblyStore/Format_V3.cs index 871a9482109..4092119a1d7 100644 --- a/tools/apput/src/AssemblyStore/Format_V3.cs +++ b/tools/apput/src/AssemblyStore/Format_V3.cs @@ -1,5 +1,17 @@ +using System.IO; + namespace ApplicationUtility; -class Format_V3 +class Format_V3 : FormatBase { + public Format_V3 (Stream storeStream, string? description) + : base (storeStream, description) + {} + + public override IAspectState Validate () + { + // TODO: validate that the store has correct format and populate the state below accordingly + // to save time later. + return new AssemblyStoreAspectState (true); + } } diff --git a/tools/apput/src/Common/BasicAspectState.cs b/tools/apput/src/Common/BasicAspectState.cs new file mode 100644 index 00000000000..d88fe1ec88f --- /dev/null +++ b/tools/apput/src/Common/BasicAspectState.cs @@ -0,0 +1,11 @@ +namespace ApplicationUtility; + +class BasicAspectState : IAspectState +{ + public bool Success { get; } + + public BasicAspectState (bool success) + { + Success = success; + } +} diff --git a/tools/apput/src/Common/IAspect.cs b/tools/apput/src/Common/IAspect.cs index 62654673f3f..73182a2d6bf 100644 --- a/tools/apput/src/Common/IAspect.cs +++ b/tools/apput/src/Common/IAspect.cs @@ -23,12 +23,12 @@ public interface IAspect /// `false` otherwise. The parameter can be anything that makes /// sense for the given aspect (e.g. a file name). /// - static bool ProbeAspect (Stream stream, string? description = null) => throw new NotImplementedException (); + static IAspectState ProbeAspect (Stream stream, string? description = null) => throw new NotImplementedException (); /// /// Load the aspect and return instance of a class implementing support for it. /// The parameter can be anything that makes /// sense for the given aspect (e.g. a file name). /// - static IAspect LoadAspect (Stream stream, string? description = null) => throw new NotImplementedException (); + static IAspect LoadAspect (Stream stream, IAspectState state, string? description = null) => throw new NotImplementedException (); } diff --git a/tools/apput/src/Common/IAspectState.cs b/tools/apput/src/Common/IAspectState.cs new file mode 100644 index 00000000000..554be193fb1 --- /dev/null +++ b/tools/apput/src/Common/IAspectState.cs @@ -0,0 +1,15 @@ +namespace ApplicationUtility; + +/// +/// An empty interface which can be used by the aspect detection mechanism to +/// preserve some state between and +/// calls, to optimize resource usage. +/// +public interface IAspectState +{ + /// + /// Indicates that whatever method returned instance of this interface, the operation was + /// successful if `true`. + /// + bool Success { get; } +} diff --git a/tools/apput/src/Detector.cs b/tools/apput/src/Detector.cs index 61495c4bf42..c9248e46e76 100644 --- a/tools/apput/src/Detector.cs +++ b/tools/apput/src/Detector.cs @@ -38,23 +38,23 @@ class Detector static IAspect? TryFindAspect (Stream stream, string? description) { - var args = new object?[] { stream, description }; var flags = BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static; foreach (Type aspect in KnownTopLevelAspects) { Log.Debug ($"Probing aspect: {aspect}"); object? result = aspect.InvokeMember ( - "ProbeAspect", flags, null, null, args + "ProbeAspect", flags, null, null, new object?[] { stream, description } ); - if (result == null || (result is bool success && !success)) { + var state = result as IAspectState; + if (state == null || !state.Success) { continue; } Log.Debug ($"Loading aspect: {aspect}"); result = aspect.InvokeMember ( - "LoadAspect", flags, null, null, args + "LoadAspect", flags, null, null, new object?[] { stream, state, description } ); if (result != null) { return (IAspect)result; diff --git a/tools/apput/src/Native/SharedLibrary.cs b/tools/apput/src/Native/SharedLibrary.cs index ce5acd17515..ca5df68b31d 100644 --- a/tools/apput/src/Native/SharedLibrary.cs +++ b/tools/apput/src/Native/SharedLibrary.cs @@ -1,10 +1,16 @@ using System; using System.IO; +using System.Text; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; using ApplicationUtility; -public class SharedLibrary : IAspect +public class SharedLibrary : IAspect, IDisposable { + const uint ELF_MAGIC = 0x464c457f; + public static string AspectName { get; } = "Native shared library"; public bool HasAndroidPayload => payloadSize > 0; @@ -13,27 +19,33 @@ public class SharedLibrary : IAspect readonly ulong payloadOffset; readonly ulong payloadSize; readonly string libraryName; + readonly bool is64Bit; + readonly Stream libraryStream; + IELF elf; + bool disposed; SharedLibrary (Stream stream, string libraryName) { - (payloadOffset, payloadSize) = FindAndroidPayload (stream); + this.libraryStream = stream; this.libraryName = libraryName; + (elf, is64Bit) = LoadELF (stream, libraryName); + (payloadOffset, payloadSize) = FindAndroidPayload (elf); } - public static IAspect LoadAspect (Stream stream, string? description) + public static IAspect LoadAspect (Stream stream, IAspectState? state, string? description) { if (String.IsNullOrEmpty (description)) { throw new ArgumentException ("Must be a shared library name", nameof (description)); } - if (!IsELFSharedLibrary (stream, description)) { - throw new InvalidOperationException ("Stream is not an ELF shared library"); + if (!IsSupportedELFSharedLibrary (stream, description)) { + throw new InvalidOperationException ("Stream is not a supported ELF shared library"); } return new SharedLibrary (stream, description); } - public static bool ProbeAspect (Stream stream, string? description) => IsELFSharedLibrary (stream, description); + public static IAspectState ProbeAspect (Stream stream, string? description) => new BasicAspectState (IsSupportedELFSharedLibrary (stream, description)); /// /// If the library has .NET for Android payload section, this @@ -42,8 +54,8 @@ public static IAspect LoadAspect (Stream stream, string? description) /// public void CopyAndroidPayload (Stream dest) { - Stream payload = OpenAndroidPayload (); - throw new NotImplementedException (); + using Stream payload = OpenAndroidPayload (); + payload.CopyTo (dest); } /// @@ -57,16 +69,139 @@ public Stream OpenAndroidPayload () throw new InvalidOperationException ("Payload section not found"); } - throw new NotImplementedException (); + if (payloadOffset > Int64.MaxValue) { + throw new InvalidOperationException ($"Payload offset of {payloadOffset} is too large to support."); + } + + if (payloadSize > Int64.MaxValue) { + throw new InvalidOperationException ($"Payload offset of {payloadSize} is too large to support."); + } + + return new SharedLibraryPayloadStream (libraryStream, (long)payloadOffset, (long)payloadSize); + } + + static bool IsSupportedELFSharedLibrary (Stream stream, string? description) + { + if (stream.Length < 4) { // Less than that and we know there isn't room for ELF magic + Log.Debug ($"SharedLibrary: stream ('{description}') is too short to be an ELF image."); + return false; + } + stream.Seek (0, SeekOrigin.Begin); + + using var reader = new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); + uint magic = reader.ReadUInt32 (); + if (magic != ELF_MAGIC) { + Log.Debug ($"SharedLibrary: stream ('{description}') is not an ELF image."); + return false; + } + stream.Seek (0, SeekOrigin.Begin); + + Class elfClass = ELFReader.CheckELFType (stream); + if (elfClass == Class.NotELF) { + Log.Debug ($"SharedLibrary: stream ('{description}') is not a supported ELF class."); + return false; + } + + if (!ELFReader.TryLoad (stream, shouldOwnStream: false, out IELF? elf) || elf == null) { + Log.Debug ($"SharedLibrary: stream ('{description}') failed to load as an ELF image while checking support."); + return false; + } + + if (elf.Type != FileType.SharedObject) { + Log.Debug ($"SharedLibrary: stream ('{description}') is not an ELF shared library image."); + return false; + } + + if (elf.Endianess != ELFSharp.Endianess.LittleEndian) { + Log.Debug ($"SharedLibrary: stream ('{description}') is not a little-endian ELF image."); + return false; + } + + bool supported = elf.Machine switch { + Machine.ARM => true, + Machine.Intel386 => true, + Machine.AArch64 => true, + Machine.AMD64 => true, + _ => false + }; + + string not = supported ? String.Empty : " not"; + Log.Debug ($"SharedLibrary: stream ('{description}') is{not} a supported ELF architecture ('{elf.Machine}')"); + + elf.Dispose (); + return supported; } - static bool IsELFSharedLibrary (Stream stream, string? description) + // We assume below that stream corresponds to a valid and supported by us ELF image. This should have been asserted by + // the `LoadAspect` method. + (IELF elf, bool is64bit) LoadELF (Stream stream, string? libraryName) { - throw new NotImplementedException (); + stream.Seek (0, SeekOrigin.Begin); + if (!ELFReader.TryLoad (stream, shouldOwnStream: false, out IELF? elf) || elf == null) { + Log.Debug ($"SharedLibrary: stream ('{libraryName}') failed to load as an ELF image."); + throw new InvalidOperationException ($"Failed to load ELF library '{libraryName}'."); + } + + bool is64 = elf.Machine switch { + Machine.ARM => false, + Machine.Intel386 => false, + + Machine.AArch64 => true, + Machine.AMD64 => true, + + _ => throw new NotSupportedException ($"Unsupported ELF architecture '{elf.Machine}'") + }; + + return (elf, is64); + } + + (ulong offset, ulong size) FindAndroidPayload (IELF elf) + { + if (!elf.TryGetSection ("payload", out ISection? payloadSection)) { + Log.Debug ($"SharedLibrary: shared library '{libraryName}' doesn't have the 'payload' section."); + return (0, 0); + } + + ulong offset; + ulong size; + + if (is64Bit) { + (offset, size) = GetOffsetAndSize64 ((Section)payloadSection); + } else { + (offset, size) = GetOffsetAndSize32 ((Section)payloadSection); + } + + Log.Debug ($"SharedLibrary: found payload section at offset {offset}, size of {size} bytes."); + return (offset, size); + + (ulong offset, ulong size) GetOffsetAndSize64 (Section payload) + { + return (payload.Offset, payload.Size); + } + + (ulong offset, ulong size) GetOffsetAndSize32 (Section payload) + { + return ((ulong)payload.Offset, (ulong)payload.Size); + } + } + + protected virtual void Dispose (bool disposing) + { + if (disposed) { + return; + } + + if (disposing) { + elf?.Dispose (); + } + + disposed = true; } - (ulong offset, ulong size) FindAndroidPayload (Stream stream) + public void Dispose () { - throw new NotImplementedException (); + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose (disposing: true); + GC.SuppressFinalize (this); } } diff --git a/tools/apput/src/Native/SharedLibraryPayloadStream.cs b/tools/apput/src/Native/SharedLibraryPayloadStream.cs new file mode 100644 index 00000000000..27ec09b879d --- /dev/null +++ b/tools/apput/src/Native/SharedLibraryPayloadStream.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +class SharedLibraryPayloadStream : Stream +{ + readonly Stream baseStream; + readonly long length; + readonly long offsetInBaseStream; + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => length; + + public override long Position { + get => throw new NotSupportedException (); + set => throw new NotSupportedException (); + } + + public SharedLibraryPayloadStream (Stream baseStream, long offset, long length) + { + if (!baseStream.CanSeek) { + throw new InvalidOperationException ($"Base stream must support seeking"); + } + + if (!baseStream.CanRead) { + throw new InvalidOperationException ($"Base stream must support reading"); + } + + if (offset >= baseStream.Length) { + throw new ArgumentOutOfRangeException (nameof (offset), $"{offset} exceeds length of the base stream ({baseStream.Length})"); + } + + if (offset + length > baseStream.Length) { + throw new InvalidOperationException ($"Not enough data in base stream after offset {offset}, length of {length} bytes is too big."); + } + + this.baseStream = baseStream; + this.length = length; + offsetInBaseStream = offset; + } + + public override int Read (byte [] buffer, int offset, int count) + { + return baseStream.Read (buffer, offset, count); + } + + public override long Seek (long offset, SeekOrigin origin) + { + return baseStream.Seek (offset + offsetInBaseStream, origin); + } + + public override void Flush () + { + throw new NotSupportedException (); + } + + public override void SetLength (long value) + { + throw new NotSupportedException (); + } + + public override void Write (byte [] buffer, int offset, int count) + { + throw new NotSupportedException (); + } +} diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index 9aa4c9ccf1a..03c4caa5368 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -56,7 +56,7 @@ protected ApplicationPackage (ZipArchive zip, string? description) Description = description; } - public static IAspect LoadAspect (Stream stream, string? description) + public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) { Log.Debug ($"ApplicationPackage: opening stream ('{description}') as a ZIP archive"); ZipArchive? zip = TryOpenAsZip (stream); @@ -170,14 +170,16 @@ void TryLoadAssemblyStores () return null; } + string fullStorePath = $"{Description}@!{storePath}"; try { - if (!AssemblyStore.ProbeAspect (storeStream, storePath)) { + IAspectState state = AssemblyStore.ProbeAspect (storeStream, fullStorePath); + if (!state.Success) { Log.Debug ($"Assembly store '{storePath}' is not in a supported format"); storeStream.Close (); return null; } - return (AssemblyStore)AssemblyStore.LoadAspect (storeStream, storePath); + return (AssemblyStore)AssemblyStore.LoadAspect (storeStream, state, fullStorePath); } catch (Exception ex) { Log.Debug ($"Failed to load assembly store '{storePath}'", ex); return null; @@ -223,12 +225,12 @@ void TryLoadAndroidManifest () } } - public static bool ProbeAspect (Stream stream, string? description) + public static IAspectState ProbeAspect (Stream stream, string? description) { Log.Debug ($"ApplicationPackage: checking if stream ('{description}') is a ZIP archive"); using ZipArchive? zip = TryOpenAsZip (stream); if (zip == null) { - return false; + return new BasicAspectState (false); } Log.Debug ($"ApplicationPackage: checking if stream ('{description}') is a supported Android ZIP package"); @@ -241,11 +243,11 @@ public static bool ProbeAspect (Stream stream, string? description) } else if (IsBase (zip)) { kind = "Base"; } else { - return false; + return new BasicAspectState (false); } Log.Debug ($"ApplicationPackage: archive is {kind}"); - return true; + return new BasicAspectState (true); } static bool IsAPK (ZipArchive zip) => HasAllEntries (zip, KnownApkEntries); From 6898b51956403094cbdce189fa66311f78e7dc6a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Jul 2025 18:21:53 +0200 Subject: [PATCH 07/12] More assembly store code --- .../apput/src/AssemblyStore/AssemblyStore.cs | 9 +- .../src/AssemblyStore/AssemblyStoreAbi.cs | 12 +- .../AssemblyStore/AssemblyStoreAspectState.cs | 2 +- .../AssemblyStoreAssemblyDescriptor.cs | 4 + .../AssemblyStoreAssemblyDescriptorV2.cs | 12 ++ .../AssemblyStoreAssemblyDescriptorV3.cs | 5 + .../src/AssemblyStore/AssemblyStoreHeader.cs | 13 +- .../src/AssemblyStore/AssemblyStoreIndex.cs | 5 - .../AssemblyStoreIndexEntryV3.cs | 6 + .../src/AssemblyStore/AssemblyStoreVersion.cs | 29 +++- tools/apput/src/AssemblyStore/FormatBase.cs | 155 +++++++++++++++++- tools/apput/src/AssemblyStore/Format_V2.cs | 2 +- tools/apput/src/AssemblyStore/Format_V3.cs | 26 ++- tools/apput/src/Package/ApplicationPackage.cs | 2 +- 14 files changed, 252 insertions(+), 30 deletions(-) create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptor.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV2.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV3.cs delete mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index 916136e6688..d358aafb59e 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -67,12 +67,11 @@ static IAspectState DoProbeAspect (Stream storeStream, string? description) } uint version = reader.ReadUInt32 (); - - // We currently support version 3. Main store version is kept in the lower 16 bits of the version word - uint mainVersion = version & 0xFFFF; + var storeVersion = new AssemblyStoreVersion (version); FormatBase? validator = null; + Log.Debug ($"AssemblyStore: store format version {storeVersion.MainVersion}"); - switch (mainVersion) { + switch (storeVersion.MainVersion) { case 2: validator = new Format_V2 (storeStream, description); break; @@ -82,7 +81,7 @@ static IAspectState DoProbeAspect (Stream storeStream, string? description) break; default: - Log.Debug ($"AssemblyStore: unsupported store version: {mainVersion}"); + Log.Debug ($"AssemblyStore: unsupported store version: {storeVersion.MainVersion}"); return new BasicAspectState (false); } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs index e5a0546034d..e5e05e26aea 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAbi.cs @@ -1,10 +1,12 @@ namespace ApplicationUtility; // Values correspond to those in `xamarin-app.hh` -enum AssemblyStoreABI +enum AssemblyStoreABI : uint { - Arm64 = 0x00010000, - Arm = 0x00020000, - X86 = 0x00030000, - X64 = 0x00040000, + Unknown = 0x00000000, + + Arm64 = 0x00010000, + Arm = 0x00020000, + X86 = 0x00030000, + X64 = 0x00040000, } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs index 5f4202ab872..2ae0b6b77f1 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs @@ -5,7 +5,7 @@ namespace ApplicationUtility; class AssemblyStoreAspectState : BasicAspectState { public AssemblyStoreHeader Header { get; } - public AssemblyStoreIndex Index { get; } + public FormatBase Format { get; } public AssemblyStoreAspectState (bool success) : base (success) diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptor.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptor.cs new file mode 100644 index 00000000000..bae35595473 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptor.cs @@ -0,0 +1,4 @@ +namespace ApplicationUtility; + +abstract class AssemblyStoreAssemblyDescriptor +{} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV2.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV2.cs new file mode 100644 index 00000000000..523e8ca9df8 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV2.cs @@ -0,0 +1,12 @@ +namespace ApplicationUtility; + +class AssemblyStoreAssemblyDescriptorV2 : AssemblyStoreAssemblyDescriptor +{ + public uint MappingIndex { get; internal set; } + public uint DataOffset { get; internal set; } + public uint DataSize { get; internal set; } + public uint DebugDataOffset { get; internal set; } + public uint DebugDataSize { get; internal set; } + public uint ConfigDataOffset { get; internal set; } + public uint ConfigDataSize { get; internal set; } +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV3.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV3.cs new file mode 100644 index 00000000000..18c1ac4974e --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAssemblyDescriptorV3.cs @@ -0,0 +1,5 @@ +namespace ApplicationUtility; + +// Format is identical to V2, class exists merely for versioning consistency +class AssemblyStoreAssemblyDescriptorV3 : AssemblyStoreAssemblyDescriptorV2 +{} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs index 566c95ce46f..7bd71890da8 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs @@ -13,8 +13,13 @@ namespace ApplicationUtility; /// class AssemblyStoreHeader { - public AssemblyStoreVersion Version { get; protected set; } - public uint? EntryCount { get; protected set; } - public uint? IndexEntryCount { get; protected set; } - public uint? IndexSize { get; protected set; } + public AssemblyStoreVersion Version { get; } + public uint? EntryCount { get; internal set; } + public uint? IndexEntryCount { get; internal set; } + public uint? IndexSize { get; internal set; } + + public AssemblyStoreHeader (AssemblyStoreVersion version) + { + Version = version; + } } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs b/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs deleted file mode 100644 index 83ae5195d5e..00000000000 --- a/tools/apput/src/AssemblyStore/AssemblyStoreIndex.cs +++ /dev/null @@ -1,5 +0,0 @@ -using ApplicationUtility; - -class AssemblyStoreIndex -{ -} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs new file mode 100644 index 00000000000..7f42e5d5965 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs @@ -0,0 +1,6 @@ +namespace ApplicationUtility; + +class AssemblyStoreIndexEntryV3 +{ + +} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs index 38083a9107c..a1183e20603 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs @@ -1,15 +1,36 @@ +using System; + namespace ApplicationUtility; class AssemblyStoreVersion { + public uint RawVersion { get; } public uint MainVersion { get; } public AssemblyStoreABI ABI { get; } public bool Is64Bit { get; } - internal AssemblyStoreVersion (uint mainVersion, AssemblyStoreABI abi, bool is64Bit) + internal AssemblyStoreVersion (uint rawVersion) { - MainVersion = mainVersion; - ABI = abi; - Is64Bit = is64Bit; + RawVersion = rawVersion; + Log.Debug ($"AssemblyStoreVersion: raw version is 0x{rawVersion:x}"); + + // Main store version is kept in the lower 16 bits of the version word + MainVersion = rawVersion & 0xFFFF; + Log.Debug ($"AssemblyStoreVersion: main version is {MainVersion}"); + + // ABI is kept in the higher 15 bits of the version word + uint abi = rawVersion & 0x7FFF0000; + Log.Debug ($"AssemblyStoreVersion: raw ABI value is 0x{abi:x}"); + + if (Enum.IsDefined (typeof(AssemblyStoreABI), abi)) { + ABI = (AssemblyStoreABI)abi; + } else { + ABI = AssemblyStoreABI.Unknown; + } + Log.Debug ($"AssemblyStoreVersion: ABI is {ABI}"); + + // 64-bit flag is the leftmost bit in the word + Is64Bit = (rawVersion & 0x80000000) == 0x80000000; + Log.Debug ($"AssemblyStoreVersion: is store 64-bit? {Is64Bit}"); } } diff --git a/tools/apput/src/AssemblyStore/FormatBase.cs b/tools/apput/src/AssemblyStore/FormatBase.cs index d337f03e2fe..6652000edf7 100644 --- a/tools/apput/src/AssemblyStore/FormatBase.cs +++ b/tools/apput/src/AssemblyStore/FormatBase.cs @@ -1,17 +1,170 @@ +using System; +using System.Collections.Generic; using System.IO; +using System.Text; namespace ApplicationUtility; +/// +/// `FormatBase` class is the base class for all format-specific validators/readers. It will +/// always implement reading the current (i.e. the `main` branch) assembly store format, with +/// subclasses required to handle differences. Subclasses are expected to override the virtual +/// `Read*` methods and completely handle reading of the respective structure, without calling +/// up to the base class. +/// abstract class FormatBase { protected Stream StoreStream { get; } protected string? Description { get; } + public AssemblyStoreHeader? Header { get; protected set; } + public List? Descriptors { get; protected set; } + protected FormatBase (Stream storeStream, string? description) { this.StoreStream = storeStream; this.Description = description; } - public abstract IAspectState Validate (); + public void Read () + { + using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); + + // They can be `null` if `Validate` wasn't called for some reason. + if (Header == null && ReadHeader (reader, out AssemblyStoreHeader? header)) { + Header = header; + } + + if (Descriptors == null && ReadAssemblyDescriptors (reader, out List? descriptors)) { + Descriptors = descriptors; + } + } + + public IAspectState Validate () + { + using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); + + if (ReadHeader (reader, out AssemblyStoreHeader? header)) { + Header = header; + } + + if (ReadAssemblyDescriptors (reader, out List? descriptors)) { + Descriptors = descriptors; + } + + return ValidateInner (); + } + + protected abstract IAspectState ValidateInner (); + + protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? header) + { + header = null; + try { + header = DoReadHeader (reader); + Log.Debug ("AssemblyStore/FormatBase: read store header."); + Log.Debug ($" Raw version: 0x{header.Version.RawVersion:x}"); + Log.Debug ($" Main version: {header.Version.MainVersion}"); + Log.Debug ($" ABI: {header.Version.ABI}"); + Log.Debug ($" 64-bit: {header.Version.Is64Bit}"); + Log.Debug ($" Entry count: {header.EntryCount}"); + Log.Debug ($" Index entry count: {header.IndexEntryCount}"); + Log.Debug ($" Index size (bytes): {header.IndexSize}"); + } catch (Exception ex) { + Log.Debug ($"AssemblyStore/FormatBase: Failed to read assembly store header. Exception thrown:", ex); + return false; + } + + return header != null; + } + + AssemblyStoreHeader? DoReadHeader (BinaryReader reader) + { + StoreStream.Seek (0, SeekOrigin.Begin); + + // From src/native/clr/include/xamarin-app.hh + // + // HEADER (fixed size) + // [MAGIC] uint; value: 0x41424158 + // [FORMAT_VERSION] uint; store format version number + // [ENTRY_COUNT] uint; number of entries in the store + // [INDEX_ENTRY_COUNT] uint; number of entries in the index + // [INDEX_SIZE] uint; index size in bytes + // + + // By the time we are called, the magic number has been verified. We simply ignore it. + uint uintValue = reader.ReadUInt32 (); // magic + uintValue = reader.ReadUInt32 (); // format version + var storeVersion = new AssemblyStoreVersion (uintValue); + + uint entryCount = reader.ReadUInt32 (); + uint indexEntryCount = reader.ReadUInt32 (); + uint indexSize = reader.ReadUInt32 (); + + return new AssemblyStoreHeader (storeVersion) { + EntryCount = entryCount, + IndexEntryCount = indexEntryCount, + IndexSize = indexSize, + }; + } + + protected virtual bool ReadAssemblyDescriptors (BinaryReader reader, out List? descriptors) + { + descriptors = null; + try { + descriptors = DoReadAssemblyDescriptors (reader); + } catch (Exception ex) { + Log.Debug ($"AssemblyStore/FormatBase: failed to read assembly descriptors. Exception thrown:", ex); + return false; + } + + return descriptors != null && descriptors.Count > 0; + } + + List? DoReadAssemblyDescriptors (BinaryReader reader) + { + if (Header == null) { + Log.Debug ($"AssemblyStore/FormatBase: unable to read descriptors, header hasn't been read."); + return null; + } + + if (Header.EntryCount == null) { + Log.Debug ($"AssemblyStore/FormatBase: unable to read descriptors, header entry count hasn't been read."); + return null; + } + + ulong indexEntrySize = Header.Version.Is64Bit ? Format_V3.IndexEntrySize64 : Format_V3.IndexEntrySize32; + ulong descriptorsOffset = (ulong)(Format_V3.HeaderSize + ((Header.EntryCount * 2) * indexEntrySize)); + + if (descriptorsOffset > Int64.MaxValue) { + Log.Debug ($"AssemblyStore/FormatBase: descriptors offset exceeds the maximum value handled by System.IO.Stream"); + return null; + } + + reader.BaseStream.Seek ((long)descriptorsOffset, SeekOrigin.Begin); + var descriptors = new List (); + + for (uint i = 0; i < Header.EntryCount; i++) { + uint mappingIndex = reader.ReadUInt32 (); + uint dataOffset = reader.ReadUInt32 (); + uint dataSize = reader.ReadUInt32 (); + uint debugDataOffset = reader.ReadUInt32 (); + uint debugDataSize = reader.ReadUInt32 (); + uint configDataOffset = reader.ReadUInt32 (); + uint configDataSize = reader.ReadUInt32 (); + + var desc = new AssemblyStoreAssemblyDescriptorV3 { + MappingIndex = mappingIndex, + DataOffset = dataOffset, + DataSize = dataSize, + DebugDataOffset = debugDataOffset, + DebugDataSize = debugDataSize, + ConfigDataOffset = configDataOffset, + ConfigDataSize = configDataSize, + }; + descriptors.Add (desc); + } + + return descriptors; + } } diff --git a/tools/apput/src/AssemblyStore/Format_V2.cs b/tools/apput/src/AssemblyStore/Format_V2.cs index cceeeb72347..0550923aee9 100644 --- a/tools/apput/src/AssemblyStore/Format_V2.cs +++ b/tools/apput/src/AssemblyStore/Format_V2.cs @@ -9,7 +9,7 @@ public Format_V2 (Stream storeStream, string? description) : base (storeStream, description) {} - public override IAspectState Validate () + protected override IAspectState ValidateInner () { throw new NotImplementedException (); } diff --git a/tools/apput/src/AssemblyStore/Format_V3.cs b/tools/apput/src/AssemblyStore/Format_V3.cs index 4092119a1d7..049f4f94e74 100644 --- a/tools/apput/src/AssemblyStore/Format_V3.cs +++ b/tools/apput/src/AssemblyStore/Format_V3.cs @@ -4,14 +4,34 @@ namespace ApplicationUtility; class Format_V3 : FormatBase { + public const uint HeaderSize = 5 * sizeof(uint); + public const uint IndexEntrySize32 = sizeof(uint) + sizeof(uint) + sizeof(byte); + public const uint IndexEntrySize64 = sizeof(ulong) + sizeof(uint) + sizeof(byte); + public const uint AssemblyDescriptorSize = 7 * sizeof(uint); + public Format_V3 (Stream storeStream, string? description) : base (storeStream, description) {} - public override IAspectState Validate () + protected override IAspectState ValidateInner () { - // TODO: validate that the store has correct format and populate the state below accordingly - // to save time later. + Log.Debug ("AssemblyStore/Format_V3: validating store format."); + if (Header == null || Header.EntryCount == null || Header.IndexEntryCount == null || Header.IndexSize == null) { + return ValidationFailed ($"AssemblyStore/Format_V3: invalid header data."); + } + + if (Descriptors == null || Descriptors.Count == 0) { + return ValidationFailed ($"AssemblyStore/Format_V3: no descriptors read."); + } + + // TODO: validate stream size + // TODO: populate return new AssemblyStoreAspectState (true); + + BasicAspectState ValidationFailed (string message) + { + Log.Debug (message); + return new BasicAspectState (false); + } } } diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index 03c4caa5368..0eb1461aa88 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -181,7 +181,7 @@ void TryLoadAssemblyStores () return (AssemblyStore)AssemblyStore.LoadAspect (storeStream, state, fullStorePath); } catch (Exception ex) { - Log.Debug ($"Failed to load assembly store '{storePath}'", ex); + Log.Debug ($"Failed to load assembly store '{storePath}'. Exception thrown:", ex); return null; } } From 4976356484316dd6926f2dd16477882b225e3108 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 10 Jul 2025 16:28:57 +0200 Subject: [PATCH 08/12] AssemblyStore + Assembly reading --- .../apput/src/AssemblyStore/AssemblyStore.cs | 33 ++++- .../AssemblyStore/AssemblyStoreAspectState.cs | 9 +- .../src/AssemblyStore/AssemblyStoreHeader.cs | 4 + .../src/AssemblyStore/AssemblyStoreVersion.cs | 5 + tools/apput/src/AssemblyStore/FormatBase.cs | 60 +++++++-- tools/apput/src/AssemblyStore/Format_V2.cs | 8 ++ tools/apput/src/AssemblyStore/Format_V3.cs | 126 ++++++++++++++++-- tools/apput/src/Native/SharedLibrary.cs | 2 +- .../src/Native/SharedLibraryPayloadStream.cs | 69 ---------- 9 files changed, 218 insertions(+), 98 deletions(-) delete mode 100644 tools/apput/src/Native/SharedLibraryPayloadStream.cs diff --git a/tools/apput/src/AssemblyStore/AssemblyStore.cs b/tools/apput/src/AssemblyStore/AssemblyStore.cs index d358aafb59e..c0f42b7d5b3 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStore.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStore.cs @@ -18,10 +18,41 @@ public class AssemblyStore : IAspect public AndroidTargetArch Architecture { get; private set; } = AndroidTargetArch.None; public ulong NumberOfAssemblies => (ulong)(Assemblies?.Count ?? 0); + AssemblyStoreAspectState storeState; + string? description; + + AssemblyStore (AssemblyStoreAspectState state, string? description) + { + storeState = state; + this.description = description; + } + + bool Read () + { + if (!storeState.Format.Read ()) { + return false; + } + + foreach (ApplicationAssembly asm in storeState.Format.Assemblies) { + Assemblies.Add (asm.Name, asm); + } + + return true; + } + public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) { var storeState = state as AssemblyStoreAspectState; - throw new NotImplementedException (); + if (storeState == null) { + throw new InvalidOperationException ("Internal error: unexpected aspect state. Was ProbeAspect unsuccessful?"); + } + + var store = new AssemblyStore (storeState, description); + if (store.Read ()) { + return store; + } + + throw new InvalidOperationException ($"Failed to load assembly store '{description}'"); } public static IAspectState ProbeAspect (Stream stream, string? description) diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs index 2ae0b6b77f1..f81e4ada7ac 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreAspectState.cs @@ -4,10 +4,11 @@ namespace ApplicationUtility; class AssemblyStoreAspectState : BasicAspectState { - public AssemblyStoreHeader Header { get; } public FormatBase Format { get; } - public AssemblyStoreAspectState (bool success) - : base (success) - {} + public AssemblyStoreAspectState (FormatBase format) + : base (success: true) + { + Format = format; + } } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs index 7bd71890da8..b55065f198a 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreHeader.cs @@ -22,4 +22,8 @@ public AssemblyStoreHeader (AssemblyStoreVersion version) { Version = version; } + + internal AssemblyStoreHeader () + : this (new AssemblyStoreVersion ()) + {} } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs index a1183e20603..f59cc0d4096 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreVersion.cs @@ -9,6 +9,11 @@ class AssemblyStoreVersion public AssemblyStoreABI ABI { get; } public bool Is64Bit { get; } + internal AssemblyStoreVersion () + { + ABI = AssemblyStoreABI.Unknown; + } + internal AssemblyStoreVersion (uint rawVersion) { RawVersion = rawVersion; diff --git a/tools/apput/src/AssemblyStore/FormatBase.cs b/tools/apput/src/AssemblyStore/FormatBase.cs index 6652000edf7..8e353734c82 100644 --- a/tools/apput/src/AssemblyStore/FormatBase.cs +++ b/tools/apput/src/AssemblyStore/FormatBase.cs @@ -7,18 +7,21 @@ namespace ApplicationUtility; /// /// `FormatBase` class is the base class for all format-specific validators/readers. It will -/// always implement reading the current (i.e. the `main` branch) assembly store format, with -/// subclasses required to handle differences. Subclasses are expected to override the virtual +/// always implement reading the current (i.e. the `main` branch) assembly store format by default, +/// with subclasses required to handle differences. Subclasses are expected to override the virtual /// `Read*` methods and completely handle reading of the respective structure, without calling /// up to the base class. /// abstract class FormatBase { + protected abstract string LogTag { get; } + protected Stream StoreStream { get; } protected string? Description { get; } public AssemblyStoreHeader? Header { get; protected set; } - public List? Descriptors { get; protected set; } + public IList? Descriptors { get; protected set; } + public IList Assemblies { get; protected set; } = null!; protected FormatBase (Stream storeStream, string? description) { @@ -26,20 +29,42 @@ protected FormatBase (Stream storeStream, string? description) this.Description = description; } - public void Read () + public bool Read () { + bool success = true; using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); // They can be `null` if `Validate` wasn't called for some reason. - if (Header == null && ReadHeader (reader, out AssemblyStoreHeader? header)) { - Header = header; + if (Header == null) { + if (ReadHeader (reader, out AssemblyStoreHeader? header) && header != null) { + Header = header; + } else { + success = false; + Header = new (); + } } - if (Descriptors == null && ReadAssemblyDescriptors (reader, out List? descriptors)) { - Descriptors = descriptors; + if (Descriptors == null) { + if (ReadAssemblyDescriptors (reader, out IList? descriptors) && descriptors != null) { + Descriptors = descriptors; + } else { + success = false; + Descriptors = new List ().AsReadOnly (); + } } + + if (ReadAssemblies (reader, out IList? assemblies) && assemblies != null) { + Assemblies = assemblies; + } else { + success = false; + Assemblies = new List ().AsReadOnly (); + } + + return success; } + protected abstract bool ReadAssemblies (BinaryReader reader, out IList? assemblies); + public IAspectState Validate () { using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); @@ -48,7 +73,7 @@ public IAspectState Validate () Header = header; } - if (ReadAssemblyDescriptors (reader, out List? descriptors)) { + if (ReadAssemblyDescriptors (reader, out IList? descriptors)) { Descriptors = descriptors; } @@ -57,6 +82,12 @@ public IAspectState Validate () protected abstract IAspectState ValidateInner (); + protected BasicAspectState ValidationFailed (string message) + { + Log.Debug (message); + return new BasicAspectState (false); + } + protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? header) { header = null; @@ -108,7 +139,7 @@ protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? }; } - protected virtual bool ReadAssemblyDescriptors (BinaryReader reader, out List? descriptors) + protected virtual bool ReadAssemblyDescriptors (BinaryReader reader, out IList? descriptors) { descriptors = null; try { @@ -121,7 +152,7 @@ protected virtual bool ReadAssemblyDescriptors (BinaryReader reader, out List 0; } - List? DoReadAssemblyDescriptors (BinaryReader reader) + IList? DoReadAssemblyDescriptors (BinaryReader reader) { if (Header == null) { Log.Debug ($"AssemblyStore/FormatBase: unable to read descriptors, header hasn't been read."); @@ -165,6 +196,11 @@ protected virtual bool ReadAssemblyDescriptors (BinaryReader reader, out List ReadAssemblyNames (BinaryReader reader) + { + throw new NotImplementedException (); } } diff --git a/tools/apput/src/AssemblyStore/Format_V2.cs b/tools/apput/src/AssemblyStore/Format_V2.cs index 0550923aee9..dec1593e8b8 100644 --- a/tools/apput/src/AssemblyStore/Format_V2.cs +++ b/tools/apput/src/AssemblyStore/Format_V2.cs @@ -1,14 +1,22 @@ using System; +using System.Collections.Generic; using System.IO; namespace ApplicationUtility; class Format_V2 : FormatBase { + protected override string LogTag => "AssemblyStore/Format_V2"; + public Format_V2 (Stream storeStream, string? description) : base (storeStream, description) {} + protected override bool ReadAssemblies (BinaryReader reader, out IList? assemblies) + { + throw new NotImplementedException (); + } + protected override IAspectState ValidateInner () { throw new NotImplementedException (); diff --git a/tools/apput/src/AssemblyStore/Format_V3.cs b/tools/apput/src/AssemblyStore/Format_V3.cs index 049f4f94e74..fa9351a9297 100644 --- a/tools/apput/src/AssemblyStore/Format_V3.cs +++ b/tools/apput/src/AssemblyStore/Format_V3.cs @@ -1,37 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; +using System.Text; namespace ApplicationUtility; class Format_V3 : FormatBase { + protected override string LogTag => "AssemblyStore/Format_V3"; + public const uint HeaderSize = 5 * sizeof(uint); public const uint IndexEntrySize32 = sizeof(uint) + sizeof(uint) + sizeof(byte); public const uint IndexEntrySize64 = sizeof(ulong) + sizeof(uint) + sizeof(byte); public const uint AssemblyDescriptorSize = 7 * sizeof(uint); + ulong assemblyNamesOffset; + public Format_V3 (Stream storeStream, string? description) : base (storeStream, description) {} - protected override IAspectState ValidateInner () + protected bool EnsureValidState (string where, out IAspectState? retval) { - Log.Debug ("AssemblyStore/Format_V3: validating store format."); + retval = null; if (Header == null || Header.EntryCount == null || Header.IndexEntryCount == null || Header.IndexSize == null) { - return ValidationFailed ($"AssemblyStore/Format_V3: invalid header data."); + retval = ValidationFailed ($"{LogTag}: invalid header data in {where}."); + return false; } if (Descriptors == null || Descriptors.Count == 0) { - return ValidationFailed ($"AssemblyStore/Format_V3: no descriptors read."); + retval = ValidationFailed ($"{LogTag}: no descriptors read in {where}."); + return false; } - // TODO: validate stream size - // TODO: populate - return new AssemblyStoreAspectState (true); + return true; + } - BasicAspectState ValidationFailed (string message) - { - Log.Debug (message); - return new BasicAspectState (false); + protected override IAspectState ValidateInner () + { + Log.Debug ($"{LogTag}: validating store format."); + if (!EnsureValidState (nameof (ValidateInner), out IAspectState? retval)) { + return retval!; } + + // Repetitive to `EnsureValidState`, but it's better than using `!` all over the place below... + Debug.Assert (Header != null); + Debug.Assert (Header.EntryCount != null); + Debug.Assert (Header.IndexEntryCount != null); + Debug.Assert (Descriptors != null); + + ulong indexEntrySize = Header.Version.Is64Bit ? IndexEntrySize64 : IndexEntrySize32; + ulong indexSize = (indexEntrySize * (ulong)Header.IndexEntryCount!); + ulong descriptorsSize = AssemblyDescriptorSize * (ulong)Header.EntryCount!; + ulong requiredStreamSize = HeaderSize + indexSize + descriptorsSize; + + // It points to the start of the assembly names block + assemblyNamesOffset = requiredStreamSize; + + // This is a trick to avoid having to read all the assembly names, but if the stream is valid, it won't be a + // problem and otherwise, well, we're validating after all. First descriptor's data offset points to the next + // byte after assembly names block. + ulong assemblyNamesSize = ((AssemblyStoreAssemblyDescriptorV3)Descriptors[0]).DataOffset - requiredStreamSize; + requiredStreamSize += assemblyNamesSize; + + foreach (var d in Descriptors) { + var desc = (AssemblyStoreAssemblyDescriptorV3)d; + + requiredStreamSize += desc.DataSize + desc.DebugDataSize + desc.ConfigDataSize; + } + Log.Debug ($"{LogTag}: calculated the required stream size to be {requiredStreamSize}"); + + if (requiredStreamSize > Int64.MaxValue) { + return ValidationFailed ($"{LogTag}: required stream size is too long for the stream API to handle."); + } + + if ((long)requiredStreamSize != StoreStream.Length) { + return ValidationFailed ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead."); + } else { + Log.Debug ($"{LogTag}: stream size is valid."); + } + + return new AssemblyStoreAspectState (this); + } + + protected override IList ReadAssemblyNames (BinaryReader reader) + { + Debug.Assert (Header != null); + Debug.Assert (Header.EntryCount != null); + + reader.BaseStream.Seek ((long)assemblyNamesOffset, SeekOrigin.Begin); + var ret = new List (); + + for (ulong i = 0; i < Header.EntryCount; i++) { + uint length = reader.ReadUInt32 (); + if (length == 0) { + continue; + } + + byte[] nameBytes = reader.ReadBytes ((int)length); + ret.Add (Encoding.UTF8.GetString (nameBytes)); + } + + return ret.AsReadOnly (); + } + + protected override bool ReadAssemblies (BinaryReader reader, out IList? assemblies) + { + Debug.Assert (Descriptors != null); + + assemblies = null; + if (!EnsureValidState (nameof (ReadAssemblies), out _)) { + return false; + } + + IList assemblyNames = ReadAssemblyNames (reader); + if (assemblyNames.Count != Descriptors.Count) { + Log.Debug ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})"); + return false; + } + + var ret = new List (); + for (int i = 0; i < Descriptors.Count; i++) { + var desc = (AssemblyStoreAssemblyDescriptorV3)Descriptors[i]; + string name = assemblyNames[i]; + var assemblyStream = new SubStream (reader.BaseStream, (long)desc.DataOffset, (long)desc.DataSize); + IAspectState assemblyState = ApplicationAssembly.ProbeAspect (assemblyStream, name); + if (!assemblyState.Success) { + assemblyStream.Dispose (); + continue; + } + + var assembly = (ApplicationAssembly)ApplicationAssembly.LoadAspect (assemblyStream, assemblyState, name); + ret.Add (assembly); + } + + assemblies = ret.AsReadOnly (); + return true; } } diff --git a/tools/apput/src/Native/SharedLibrary.cs b/tools/apput/src/Native/SharedLibrary.cs index ca5df68b31d..64ba7490ad2 100644 --- a/tools/apput/src/Native/SharedLibrary.cs +++ b/tools/apput/src/Native/SharedLibrary.cs @@ -77,7 +77,7 @@ public Stream OpenAndroidPayload () throw new InvalidOperationException ($"Payload offset of {payloadSize} is too large to support."); } - return new SharedLibraryPayloadStream (libraryStream, (long)payloadOffset, (long)payloadSize); + return new SubStream (libraryStream, (long)payloadOffset, (long)payloadSize); } static bool IsSupportedELFSharedLibrary (Stream stream, string? description) diff --git a/tools/apput/src/Native/SharedLibraryPayloadStream.cs b/tools/apput/src/Native/SharedLibraryPayloadStream.cs deleted file mode 100644 index 27ec09b879d..00000000000 --- a/tools/apput/src/Native/SharedLibraryPayloadStream.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.IO; - -namespace ApplicationUtility; - -class SharedLibraryPayloadStream : Stream -{ - readonly Stream baseStream; - readonly long length; - readonly long offsetInBaseStream; - - public override bool CanRead => true; - public override bool CanSeek => true; - public override bool CanWrite => false; - public override long Length => length; - - public override long Position { - get => throw new NotSupportedException (); - set => throw new NotSupportedException (); - } - - public SharedLibraryPayloadStream (Stream baseStream, long offset, long length) - { - if (!baseStream.CanSeek) { - throw new InvalidOperationException ($"Base stream must support seeking"); - } - - if (!baseStream.CanRead) { - throw new InvalidOperationException ($"Base stream must support reading"); - } - - if (offset >= baseStream.Length) { - throw new ArgumentOutOfRangeException (nameof (offset), $"{offset} exceeds length of the base stream ({baseStream.Length})"); - } - - if (offset + length > baseStream.Length) { - throw new InvalidOperationException ($"Not enough data in base stream after offset {offset}, length of {length} bytes is too big."); - } - - this.baseStream = baseStream; - this.length = length; - offsetInBaseStream = offset; - } - - public override int Read (byte [] buffer, int offset, int count) - { - return baseStream.Read (buffer, offset, count); - } - - public override long Seek (long offset, SeekOrigin origin) - { - return baseStream.Seek (offset + offsetInBaseStream, origin); - } - - public override void Flush () - { - throw new NotSupportedException (); - } - - public override void SetLength (long value) - { - throw new NotSupportedException (); - } - - public override void Write (byte [] buffer, int offset, int count) - { - throw new NotSupportedException (); - } -} From 1c979eed122c68a8875bf15bcca17cbb54795103 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 11 Jul 2025 12:19:19 +0200 Subject: [PATCH 09/12] Oops, forgot to commit this one --- tools/apput/src/Common/SubStream.cs | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tools/apput/src/Common/SubStream.cs diff --git a/tools/apput/src/Common/SubStream.cs b/tools/apput/src/Common/SubStream.cs new file mode 100644 index 00000000000..4539489ad2f --- /dev/null +++ b/tools/apput/src/Common/SubStream.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +class SubStream : Stream +{ + readonly Stream baseStream; + readonly long length; + readonly long offsetInParentStream; + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => length; + + public override long Position { + get => throw new NotSupportedException (); + set => throw new NotSupportedException (); + } + + public SubStream (Stream baseStream, long offsetInParentStream, long length) + { + if (!baseStream.CanSeek) { + throw new InvalidOperationException ($"Base stream must support seeking"); + } + + if (!baseStream.CanRead) { + throw new InvalidOperationException ($"Base stream must support reading"); + } + + if (offsetInParentStream >= baseStream.Length) { + throw new ArgumentOutOfRangeException (nameof (offsetInParentStream), $"{offsetInParentStream} exceeds length of the base stream ({baseStream.Length})"); + } + + if (offsetInParentStream + length > baseStream.Length) { + throw new InvalidOperationException ($"Not enough data in base stream after offset {offsetInParentStream}, length of {length} bytes is too big."); + } + + this.baseStream = baseStream; + this.length = length; + this.offsetInParentStream = offsetInParentStream; + } + + public override int Read (byte [] buffer, int offset, int count) + { + return baseStream.Read (buffer, offset, count); + } + + public override long Seek (long offset, SeekOrigin origin) + { + return baseStream.Seek (offset + offsetInParentStream, origin); + } + + public override void Flush () + { + throw new NotSupportedException (); + } + + public override void SetLength (long value) + { + throw new NotSupportedException (); + } + + public override void Write (byte [] buffer, int offset, int count) + { + throw new NotSupportedException (); + } +} From 21f1c5e69beeca7bf7eaa3410a068659fcfdc198 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 11 Jul 2025 16:56:40 +0200 Subject: [PATCH 10/12] More assembly store + assemblies --- .../ApplicationAssembly.cs | 125 +++++++++++++++++- .../AssemblyStore/AssemblyStoreIndexEntry.cs | 4 + .../AssemblyStoreIndexEntryV3.cs | 11 +- tools/apput/src/AssemblyStore/FormatBase.cs | 10 +- tools/apput/src/AssemblyStore/Format_V3.cs | 76 +++++++++-- tools/apput/src/Common/Utilities.cs | 13 ++ 6 files changed, 215 insertions(+), 24 deletions(-) create mode 100644 tools/apput/src/AssemblyStore/AssemblyStoreIndexEntry.cs diff --git a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs index 48f4416a0b6..b44de13deb0 100644 --- a/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs +++ b/tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs @@ -5,21 +5,134 @@ namespace ApplicationUtility; public class ApplicationAssembly : IAspect { + const string LogTag = "ApplicationAssembly"; + const uint COMPRESSED_MAGIC = 0x5A4C4158; // 'XALZ', little-endian + const ushort MSDOS_EXE_MAGIC = 0x5A4D; // 'MZ' + const uint PE_EXE_MAGIC = 0x00004550; // 'PE\0\0' + public static string AspectName { get; } = "Application assembly"; - public bool IsCompressed { get; private set; } - public string Name { get; private set; } = ""; - public ulong CompressedSize { get; private set; } - public ulong Size { get; private set; } - public bool IgnoreOnLoad { get; private set; } + public bool IsCompressed { get; } + public string Name { get; } + public ulong CompressedSize { get; } + public ulong Size { get; } + public bool IgnoreOnLoad { get; } + public ulong NameHash { get; internal set; } + + readonly Stream? assemblyStream; + + ApplicationAssembly (Stream stream, uint uncompressedSize, string? description, bool isCompressed) + { + assemblyStream = stream; + Size = uncompressedSize; + CompressedSize = isCompressed ? (ulong)stream.Length : 0; + IsCompressed = isCompressed; + Name = NameMe (description); + } + + ApplicationAssembly (string? description, bool isIgnored) + { + IgnoreOnLoad = isIgnored; + Name = NameMe (description); + } + + static string NameMe (string? description) => String.IsNullOrEmpty (description) ? "Unnamed" : description; + + // This is a special case, as much as I hate to have one. Ignored assemblies exist only in the assembly store's + // index. They have an associated descriptor, but no data whatsoever. For that reason, we can't go the `ProbeAspect` + // + `LoadAspect` route, so `AssemblyStore` will call this method for them. + public static IAspect CreateIgnoredAssembly (string? description, ulong nameHash) + { + Log.Debug ($"{LogTag}: stream ('{description}') is an ignored assembly."); + return new ApplicationAssembly (description, isIgnored: true) { + NameHash = nameHash, + }; + } public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) { - throw new NotImplementedException (); + using var reader = Utilities.GetReaderAndRewindStream (stream); + if (ReadCompressedHeader (reader, out uint uncompressedLength)) { + return new ApplicationAssembly (stream, uncompressedLength, description, isCompressed: true); + } + + return new ApplicationAssembly (stream, (uint)stream.Length, description, isCompressed: false); } public static IAspectState ProbeAspect (Stream stream, string? description) + { + Log.Debug ($"{LogTag}: probing stream ('{description}')"); + if (stream.Length == 0) { + // It can happen if the assembly store index or name table are corrupted and we cannot + // determine if an assembly is ignored or not. If it is ignored, it will have no data + // available and so the stream will have length of 0 + return new BasicAspectState (false); + } + + // If we detect compressed assembly signature, we won't proceed with checking whether + // the rest of data is actually a valid managed assembly. This is to avoid doing a + // costly operation of decompressing when e.g. loading data from an assemblystore, when + // we potentially create a lot of `ApplicationAssembly` instances. Presence of the compression + // header is enough for the probing stage. + + using var reader = Utilities.GetReaderAndRewindStream (stream); + if (ReadCompressedHeader (reader, out _)) { + Log.Debug ($"{LogTag}: stream ('{description}') is a compressed assembly."); + return new BasicAspectState (true); + } + + // We could use PEReader (https://learn.microsoft.com/en-us/dotnet/api/system.reflection.portableexecutable.pereader) + // but it would be too heavy for our purpose here. + reader.BaseStream.Seek (0, SeekOrigin.Begin); + ushort mzExeMagic = reader.ReadUInt16 (); + if (mzExeMagic != MSDOS_EXE_MAGIC) { + return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have MS-DOS executable signature."); + } + + const long PE_HEADER_OFFSET = 0x3c; + if (reader.BaseStream.Length <= PE_HEADER_OFFSET) { + return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted MS-DOS executable image (too short, offset {PE_HEADER_OFFSET} is bigger than stream size)."); + } + + // Offset at 0x3C is where we can read the 32-bit offset to the PE header + reader.BaseStream.Seek (PE_HEADER_OFFSET, SeekOrigin.Begin); + uint uintVal = reader.ReadUInt32 (); + if (reader.BaseStream.Length <= (long)uintVal) { + return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted PE executable image (too short, offset {uintVal} is bigger than stream size)."); + } + + reader.BaseStream.Seek ((long)uintVal, SeekOrigin.Begin); + uintVal = reader.ReadUInt32 (); + if (uintVal != PE_EXE_MAGIC) { + return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have PE executable signature."); + } + // This is good enough for us + + Log.Debug ($"{LogTag}: stream ('{description}') appears to be a PE image."); + return new BasicAspectState (true); + } + + /// + /// Writes assembly data to the indicated file, uncompressing it if necessary. If the destination + /// file exists, it will be overwritten. + /// + public void SaveToFile (string filePath) { throw new NotImplementedException (); } + + // We don't care about the descriptor index here, it's only needed during the run time + static bool ReadCompressedHeader (BinaryReader reader, out uint uncompressedLength) + { + uncompressedLength = 0; + + uint uintVal = reader.ReadUInt32 (); + if (uintVal != COMPRESSED_MAGIC) { + return false; + } + + uintVal = reader.ReadUInt32 (); // descriptor index + uncompressedLength = reader.ReadUInt32 (); + return true; + } } diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntry.cs b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntry.cs new file mode 100644 index 00000000000..743fa3c9556 --- /dev/null +++ b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntry.cs @@ -0,0 +1,4 @@ +namespace ApplicationUtility; + +abstract class AssemblyStoreIndexEntry +{} diff --git a/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs index 7f42e5d5965..590b6e3b9ac 100644 --- a/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs +++ b/tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs @@ -1,6 +1,15 @@ namespace ApplicationUtility; -class AssemblyStoreIndexEntryV3 +class AssemblyStoreIndexEntryV3 : AssemblyStoreIndexEntry { + public ulong NameHash { get; } + public uint DescriptorIndex { get; } + public bool Ignore { get; } + public AssemblyStoreIndexEntryV3 (ulong nameHash, uint descriptorIndex, byte ignore) + { + NameHash = nameHash; + DescriptorIndex = descriptorIndex; + Ignore = ignore != 0; + } } diff --git a/tools/apput/src/AssemblyStore/FormatBase.cs b/tools/apput/src/AssemblyStore/FormatBase.cs index 8e353734c82..3db5b50043c 100644 --- a/tools/apput/src/AssemblyStore/FormatBase.cs +++ b/tools/apput/src/AssemblyStore/FormatBase.cs @@ -32,7 +32,7 @@ protected FormatBase (Stream storeStream, string? description) public bool Read () { bool success = true; - using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); + using var reader = Utilities.GetReaderAndRewindStream (StoreStream); // They can be `null` if `Validate` wasn't called for some reason. if (Header == null) { @@ -67,7 +67,7 @@ public bool Read () public IAspectState Validate () { - using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true); + using var reader = Utilities.GetReaderAndRewindStream (StoreStream); if (ReadHeader (reader, out AssemblyStoreHeader? header)) { Header = header; @@ -82,12 +82,6 @@ public IAspectState Validate () protected abstract IAspectState ValidateInner (); - protected BasicAspectState ValidationFailed (string message) - { - Log.Debug (message); - return new BasicAspectState (false); - } - protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? header) { header = null; diff --git a/tools/apput/src/AssemblyStore/Format_V3.cs b/tools/apput/src/AssemblyStore/Format_V3.cs index fa9351a9297..352081a02bf 100644 --- a/tools/apput/src/AssemblyStore/Format_V3.cs +++ b/tools/apput/src/AssemblyStore/Format_V3.cs @@ -4,6 +4,8 @@ using System.IO; using System.Text; +using Xamarin.Android.Tasks; + namespace ApplicationUtility; class Format_V3 : FormatBase @@ -25,12 +27,12 @@ protected bool EnsureValidState (string where, out IAspectState? retval) { retval = null; if (Header == null || Header.EntryCount == null || Header.IndexEntryCount == null || Header.IndexSize == null) { - retval = ValidationFailed ($"{LogTag}: invalid header data in {where}."); + retval = Utilities.GetFailureAspectState ($"{LogTag}: invalid header data in {where}."); return false; } if (Descriptors == null || Descriptors.Count == 0) { - retval = ValidationFailed ($"{LogTag}: no descriptors read in {where}."); + retval = Utilities.GetFailureAspectState ($"{LogTag}: no descriptors read in {where}."); return false; } @@ -47,11 +49,12 @@ protected override IAspectState ValidateInner () // Repetitive to `EnsureValidState`, but it's better than using `!` all over the place below... Debug.Assert (Header != null); Debug.Assert (Header.EntryCount != null); + Debug.Assert (Header.IndexSize != null); Debug.Assert (Header.IndexEntryCount != null); Debug.Assert (Descriptors != null); ulong indexEntrySize = Header.Version.Is64Bit ? IndexEntrySize64 : IndexEntrySize32; - ulong indexSize = (indexEntrySize * (ulong)Header.IndexEntryCount!); + ulong indexSize = (ulong)Header.IndexSize; // (indexEntrySize * (ulong)Header.IndexEntryCount!); ulong descriptorsSize = AssemblyDescriptorSize * (ulong)Header.EntryCount!; ulong requiredStreamSize = HeaderSize + indexSize + descriptorsSize; @@ -72,11 +75,11 @@ protected override IAspectState ValidateInner () Log.Debug ($"{LogTag}: calculated the required stream size to be {requiredStreamSize}"); if (requiredStreamSize > Int64.MaxValue) { - return ValidationFailed ($"{LogTag}: required stream size is too long for the stream API to handle."); + return Utilities.GetFailureAspectState ($"{LogTag}: required stream size is too long for the stream API to handle."); } if ((long)requiredStreamSize != StoreStream.Length) { - return ValidationFailed ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead."); + return Utilities.GetFailureAspectState ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead."); } else { Log.Debug ($"{LogTag}: stream size is valid."); } @@ -107,6 +110,8 @@ protected override IList ReadAssemblyNames (BinaryReader reader) protected override bool ReadAssemblies (BinaryReader reader, out IList? assemblies) { + Debug.Assert (Header != null); + Debug.Assert (Header.IndexEntryCount != null); Debug.Assert (Descriptors != null); assemblies = null; @@ -115,16 +120,42 @@ protected override bool ReadAssemblies (BinaryReader reader, out IList assemblyNames = ReadAssemblyNames (reader); - if (assemblyNames.Count != Descriptors.Count) { - Log.Debug ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})"); - return false; + bool assemblyNamesUnreliable = assemblyNames.Count != Descriptors.Count; + if (assemblyNamesUnreliable) { + Log.Error ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})"); + } + + bool is64Bit = Header.Version.Is64Bit; + var index = new Dictionary (); + reader.BaseStream.Seek ((long)HeaderSize, SeekOrigin.Begin); + for (uint i = 0; i < Header.IndexEntryCount; i++) { + ulong hash = is64Bit ? reader.ReadUInt64 () : reader.ReadUInt32 (); + uint descIdx = reader.ReadUInt32 (); + byte ignore = reader.ReadByte (); + + if (index.ContainsKey (hash)) { + Log.Error ($"{LogTag}: duplicate assembly name hash (0x{hash:x}) found in the '{Description}' assembly store."); + continue; + } + Log.Debug ($"{LogTag}: index entry {i} hash == 0x{hash:x}"); + index.Add (hash, new AssemblyStoreIndexEntryV3 (hash, descIdx, ignore)); } var ret = new List (); for (int i = 0; i < Descriptors.Count; i++) { var desc = (AssemblyStoreAssemblyDescriptorV3)Descriptors[i]; - string name = assemblyNames[i]; + string name = assemblyNamesUnreliable ? "" : assemblyNames[i]; var assemblyStream = new SubStream (reader.BaseStream, (long)desc.DataOffset, (long)desc.DataSize); + + ulong hash = NameHash (name); + Log.Debug ($"{LogTag}: hash for assembly '{name}' is 0x{hash:x}"); + + bool isIgnored = CheckIgnored (hash); + if (isIgnored) { + ret.Add ((ApplicationAssembly)ApplicationAssembly.CreateIgnoredAssembly (name, hash)); + continue; + } + IAspectState assemblyState = ApplicationAssembly.ProbeAspect (assemblyStream, name); if (!assemblyState.Success) { assemblyStream.Dispose (); @@ -132,10 +163,37 @@ protected override bool ReadAssemblies (BinaryReader reader, out IList Date: Mon, 14 Jul 2025 18:45:00 +0200 Subject: [PATCH 11/12] Android binary XML parser (for in-package AndroidManifest.xml) A big buggy still, TBC --- tools/apput/src/Android/ARSCHeader.cs | 60 +++ tools/apput/src/Android/AXMLParser.cs | 378 ++++++++++++++++++ tools/apput/src/Android/AndroidManifest.cs | 58 +++ .../src/Android/AndroidManifestAspectState.cs | 12 + .../Android/AndroidManifestAttributeType.cs | 52 +++ .../src/Android/AndroidManifestChunkType.cs | 23 ++ .../src/Android/AndroidManifestStringBlock.cs | 176 ++++++++ tools/apput/src/Detector.cs | 12 +- tools/apput/src/Package/ApplicationPackage.cs | 36 +- 9 files changed, 799 insertions(+), 8 deletions(-) create mode 100644 tools/apput/src/Android/ARSCHeader.cs create mode 100644 tools/apput/src/Android/AXMLParser.cs create mode 100644 tools/apput/src/Android/AndroidManifest.cs create mode 100644 tools/apput/src/Android/AndroidManifestAspectState.cs create mode 100644 tools/apput/src/Android/AndroidManifestAttributeType.cs create mode 100644 tools/apput/src/Android/AndroidManifestChunkType.cs create mode 100644 tools/apput/src/Android/AndroidManifestStringBlock.cs diff --git a/tools/apput/src/Android/ARSCHeader.cs b/tools/apput/src/Android/ARSCHeader.cs new file mode 100644 index 00000000000..446d831ad8a --- /dev/null +++ b/tools/apput/src/Android/ARSCHeader.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +class ARSCHeader +{ + // This is the minimal size such a header must have. There might be other header data too! + const long MinimumSize = 2 + 2 + 4; + + readonly long start; + readonly uint size; + readonly ushort type; + readonly ushort headerSize; + readonly bool unknownType; + + public AndroidManifestChunkType Type => unknownType ? AndroidManifestChunkType.Null : (AndroidManifestChunkType)type; + public ushort TypeRaw => type; + public ushort HeaderSize => headerSize; + public uint Size => size; + public long End => start + (long)size; + + public ARSCHeader (Stream data, AndroidManifestChunkType? expectedType = null) + { + start = data.Position; + if (data.Length < start + MinimumSize) { + throw new InvalidDataException ($"Input data not large enough. Offset: {start}"); + } + + // Data in AXML is little-endian, which is fortuitous as that's the only format BinaryReader understands. + using BinaryReader reader = Utilities.GetReaderAndRewindStream (data); + + // ushort: type + // ushort: header_size + // uint: size + type = reader.ReadUInt16 (); + headerSize = reader.ReadUInt16 (); + + // Total size of the chunk, including the header + size = reader.ReadUInt32 (); + + if (expectedType != null && type != (ushort)expectedType) { + throw new InvalidOperationException ($"Header type is not equal to the expected type ({expectedType}): got 0x{type:x}, expected 0x{(ushort)expectedType:x}"); + } + + unknownType = !Enum.IsDefined (typeof(AndroidManifestChunkType), type); + + if (headerSize < MinimumSize) { + throw new InvalidDataException ($"Declared header size is smaller than required size of {MinimumSize}. Offset: {start}"); + } + + if (size < MinimumSize) { + throw new InvalidDataException ($"Declared chunk size is smaller than required size of {MinimumSize}. Offset: {start}"); + } + + if (size < headerSize) { + throw new InvalidDataException ($"Declared chunk size ({size}) is smaller than header size ({headerSize})! Offset: {start}"); + } + } +} diff --git a/tools/apput/src/Android/AXMLParser.cs b/tools/apput/src/Android/AXMLParser.cs new file mode 100644 index 00000000000..80c9151c6aa --- /dev/null +++ b/tools/apput/src/Android/AXMLParser.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Xml; + +namespace ApplicationUtility; + +// +// Based on https://github.com/androguard/androguard/tree/832104db3eb5dc3cc66b30883fa8ce8712dfa200/androguard/core/axml +// +class AXMLParser +{ + // Position of fields inside an attribute + const int ATTRIBUTE_IX_NAMESPACE_URI = 0; + const int ATTRIBUTE_IX_NAME = 1; + const int ATTRIBUTE_IX_VALUE_STRING = 2; + const int ATTRIBUTE_IX_VALUE_TYPE = 3; + const int ATTRIBUTE_IX_VALUE_DATA = 4; + const int ATTRIBUTE_LENGHT = 5; + + const long MinimumDataSize = 8; + const long MaximumDataSize = (long)UInt32.MaxValue; + + const uint ComplexUnitMask = 0x0f; + + static readonly float[] RadixMultipliers = { + 0.00390625f, + 3.051758E-005f, + 1.192093E-007f, + 4.656613E-010f, + }; + + static readonly string[] DimensionUnits = { + "px", + "dip", + "sp", + "pt", + "in", + "mm", + }; + + static readonly string[] FractionUnits = { + "%", + "%p", + }; + + Stream data; + long dataSize; + ARSCHeader axmlHeader; + uint fileSize; + AndroidManifestStringBlock stringPool; + bool valid = true; + long initialPosition; + + public bool IsValid => valid; + + public AXMLParser (Stream data) + { + this.data = data; + dataSize = data.Length; + + // Minimum is a single ARSCHeader, which would be a strange edge case... + if (dataSize < MinimumDataSize) { + throw new InvalidDataException ($"Input data size too small for it to be valid AXML content ({dataSize} < {MinimumDataSize})"); + } + + // This would be even stranger, if an AXML file is larger than 4GB... + // But this is not possible as the maximum chunk size is a unsigned 4 byte int. + if (dataSize > MaximumDataSize) { + throw new InvalidDataException ($"Input data size too large for it to be a valid AXML content ({dataSize} > {MaximumDataSize})"); + } + + try { + axmlHeader = new ARSCHeader (data); + } catch (Exception) { + Log.Error ("Error parsing the first data header"); + throw; + } + + if (axmlHeader.HeaderSize != 8) { + throw new InvalidDataException ($"This does not look like AXML data. header size does not equal 8. header size = {axmlHeader.Size}"); + } + + fileSize = axmlHeader.Size; + if (fileSize > dataSize) { + throw new InvalidDataException ($"This does not look like AXML data. Declared data size does not match real size: {fileSize} vs {dataSize}"); + } + + if (fileSize < dataSize) { + Log.Warning ($"Declared data size ({fileSize}) is smaller than total data size ({dataSize}). Was something appended to the file? Trying to parse it anyways."); + } + + if (axmlHeader.Type != AndroidManifestChunkType.Xml) { + Log.Warning ($"AXML file has an unusual resource type, trying to parse it anyways. Resource Type: 0x{(ushort)axmlHeader.Type:04x}"); + } + + ARSCHeader stringPoolHeader = new ARSCHeader (data, AndroidManifestChunkType.StringPool); + if (stringPoolHeader.HeaderSize != 28) { + throw new InvalidDataException ($"This does not look like an AXML file. String chunk header size does not equal 28. Header size = {stringPoolHeader.Size}"); + } + + stringPool = new AndroidManifestStringBlock (data, stringPoolHeader); + initialPosition = data.Position; + } + + public XmlDocument? Parse () + { + valid = true; + + XmlDocument ret = new XmlDocument (); + XmlDeclaration declaration = ret.CreateXmlDeclaration ("1.0", stringPool.IsUTF8 ? "UTF-8" : "UTF-16", null); + ret.InsertBefore (declaration, ret.DocumentElement); + + using var reader = Utilities.GetReaderAndRewindStream (data); + ARSCHeader? header; + string? nsPrefix = null; + string? nsUri = null; + uint prefixIndex = 0; + uint uriIndex = 0; + var nsUriToPrefix = new Dictionary (StringComparer.Ordinal); + XmlNode? currentNode = ret.DocumentElement; + + while (data.Position < dataSize) { + header = new ARSCHeader (data); + + // Special chunk: Resource Map. This chunk might follow the string pool. + if (header.Type == AndroidManifestChunkType.XmlResourceMap) { + if (!SkipOverResourceMap (header, reader)) { + valid = false; + break; + } + continue; + } + + // XML chunks + + // Skip over unknown types + if (!Enum.IsDefined (typeof(AndroidManifestChunkType), header.TypeRaw)) { + Log.Warning ($"Unknown chunk type 0x{header.TypeRaw:x} at offset {data.Position}. Skipping over {header.Size} bytes"); + data.Seek (header.Size, SeekOrigin.Current); + continue; + } + + // Check that we read a correct header + if (header.HeaderSize != 16) { + Log.Warning ($"XML chunk header size is not 16. Chunk type {header.Type} (0x{header.TypeRaw:x}), chunk size {header.Size}"); + data.Seek (header.Size, SeekOrigin.Current); + continue; + } + + // Line Number of the source file, only used as meta information + uint lineNumber = reader.ReadUInt32 (); + + // Comment_Index (usually 0xffffffff) + uint commentIndex = reader.ReadUInt32 (); + + if (commentIndex != 0xffffffff && (header.Type == AndroidManifestChunkType.XmlStartNamespace || header.Type == AndroidManifestChunkType.XmlEndNamespace)) { + Log.Warning ($"Unhandled Comment at namespace chunk: {commentIndex}"); + } + + if (header.Type == AndroidManifestChunkType.XmlStartNamespace) { + prefixIndex = reader.ReadUInt32 (); + uriIndex = reader.ReadUInt32 (); + + nsPrefix = stringPool.GetString (prefixIndex); + nsUri = stringPool.GetString (uriIndex); + + if (!String.IsNullOrEmpty (nsUri)) { + nsUriToPrefix[nsUri] = nsPrefix ?? String.Empty; + } + + Log.Debug ($"Start of Namespace mapping: prefix {prefixIndex}: '{nsPrefix}' --> uri {uriIndex}: '{nsUri}'"); + + if (String.IsNullOrEmpty (nsUri)) { + Log.Warning ($"Namespace prefix '{nsPrefix}' resolves to empty URI."); + } + + continue; + } + + if (header.Type == AndroidManifestChunkType.XmlEndNamespace) { + // Namespace handling is **really** simplified, since we expect to deal only with AndroidManifest.xml which should have just one namespace. + // There should be no problems with that. Famous last words. + uint endPrefixIndex = reader.ReadUInt32 (); + uint endUriIndex = reader.ReadUInt32 (); + + Log.Debug ($"End of Namespace mapping: prefix {endPrefixIndex}, uri {endUriIndex}"); + if (endPrefixIndex != prefixIndex) { + Log.Warning ($"Prefix index of Namespace end doesn't match the last Namespace prefix index: {prefixIndex} != {endPrefixIndex}"); + } + + if (endUriIndex != uriIndex) { + Log.Warning ($"URI index of Namespace end doesn't match the last Namespace URI index: {uriIndex} != {endUriIndex}"); + } + + string? endUri = stringPool.GetString (endUriIndex); + if (!String.IsNullOrEmpty (endUri) && nsUriToPrefix.ContainsKey (endUri)) { + nsUriToPrefix.Remove (endUri); + } + + nsPrefix = null; + nsUri = null; + prefixIndex = 0; + uriIndex = 0; + + continue; + } + + uint tagNsUriIndex; + uint tagNameIndex; + string? tagName; +// string? tagNs; // TODO: implement + + if (header.Type == AndroidManifestChunkType.XmlStartElement) { + // The TAG consists of some fields: + // * (chunk_size, line_number, comment_index - we read before) + // * namespace_uri + // * name + // * flags + // * attribute_count + // * class_attribute + // After that, there are two lists of attributes, 20 bytes each + tagNsUriIndex = reader.ReadUInt32 (); + tagNameIndex = reader.ReadUInt32 (); + uint tagFlags = reader.ReadUInt32 (); + uint attributeCount = reader.ReadUInt32 () & 0xffff; + uint classAttribute = reader.ReadUInt32 (); + + // Tag name is, of course, required but instead of throwing an exception should we find none, we use a fake name in hope that we can still salvage + // the document. + tagName = stringPool.GetString (tagNameIndex) ?? "unnamedTag"; + Log.Debug ($"Start of tag '{tagName}', NS URI index {tagNsUriIndex}"); + Log.Debug ($"Reading tag attributes ({attributeCount}):"); + + string? tagNsUri = tagNsUriIndex != 0xffffffff ? stringPool.GetString (tagNsUriIndex) : null; + string? tagNsPrefix; + + if (String.IsNullOrEmpty (tagNsUri) || !nsUriToPrefix.TryGetValue (tagNsUri, out tagNsPrefix)) { + tagNsPrefix = null; + } + + XmlElement element = ret.CreateElement (tagNsPrefix, tagName, tagNsUri); + if (currentNode == null) { + ret.AppendChild (element); + if (!String.IsNullOrEmpty (nsPrefix) && !String.IsNullOrEmpty (nsUri)) { + ret.DocumentElement!.SetAttribute ($"xmlns:{nsPrefix}", nsUri); + } + } else { + currentNode.AppendChild (element); + } + currentNode = element; + + for (uint i = 0; i < attributeCount; i++) { + uint attrNsIdx = reader.ReadUInt32 (); // string index + uint attrNameIdx = reader.ReadUInt32 (); // string index + uint attrValue = reader.ReadUInt32 (); + uint attrType = reader.ReadUInt32 () >> 24; + uint attrData = reader.ReadUInt32 (); + + string? attrNs = attrNsIdx != 0xffffffff ? stringPool.GetString (attrNsIdx) : String.Empty; + string? attrName = stringPool.GetString (attrNameIdx); + + if (String.IsNullOrEmpty (attrName)) { + Log.Warning ($"Attribute without name, ignoring. Offset: {data.Position}"); + continue; + } + + Log.Debug ($" '{attrName}': ns == '{attrNs}'; value == 0x{attrValue:x}; type == 0x{attrType:x}; data == 0x{attrData:x}"); + XmlAttribute attr; + + if (!String.IsNullOrEmpty (attrNs)) { + attr = ret.CreateAttribute (nsUriToPrefix[attrNs], attrName, attrNs); + } else { + attr = ret.CreateAttribute (attrName!); + } + attr.Value = GetAttributeValue (attrValue, attrType, attrData); + element.SetAttributeNode (attr); + } + continue; + } + + if (header.Type == AndroidManifestChunkType.XmlEndElement) { + tagNsUriIndex = reader.ReadUInt32 (); + tagNameIndex = reader.ReadUInt32 (); + + tagName = stringPool.GetString (tagNameIndex); + Log.Debug ($"End of tag '{tagName}', NS URI index {tagNsUriIndex}"); + currentNode = currentNode?.ParentNode!; + continue; + } + + // TODO: add support for CDATA + } + + return ret; + } + + string GetAttributeValue (uint attrValue, uint attrType, uint attrData) + { + if (!Enum.IsDefined (typeof(AndroidManifestAttributeType), attrType)) { + Log.Warning ($"Unknown attribute type value 0x{attrType:x}, returning empty attribute value (data == 0x{attrData:x}). Offset: {data.Position}"); + return String.Empty; + } + + switch ((AndroidManifestAttributeType)attrType) { + case AndroidManifestAttributeType.Null: + return attrData == 0 ? "?NULL?" : String.Empty; + + case AndroidManifestAttributeType.Reference: + return $"@{MaybePrefix()}{attrData:x08}"; + + case AndroidManifestAttributeType.Attribute: + return $"?{MaybePrefix()}{attrData:x08}"; + + case AndroidManifestAttributeType.String: + return stringPool.GetString (attrData) ?? String.Empty; + + case AndroidManifestAttributeType.Float: + return $"{(float)attrData}"; + + case AndroidManifestAttributeType.Dimension: + return $"{ComplexToFloat(attrData)}{DimensionUnits[attrData & ComplexUnitMask]}"; + + case AndroidManifestAttributeType.Fraction: + return $"{ComplexToFloat(attrData) * 100.0f}{FractionUnits[attrData & ComplexUnitMask]}"; + + case AndroidManifestAttributeType.IntDec: + return attrData.ToString (); + + case AndroidManifestAttributeType.IntHex: + return $"0x{attrData:X08}"; + + case AndroidManifestAttributeType.IntBoolean: + return attrData == 0 ? "false" : "true"; + + case AndroidManifestAttributeType.IntColorARGB8: + case AndroidManifestAttributeType.IntColorRGB8: + case AndroidManifestAttributeType.IntColorARGB4: + case AndroidManifestAttributeType.IntColorRGB4: + return $"#{attrData:X08}"; + } + + return String.Empty; + + string MaybePrefix () + { + if (attrData >> 24 == 1) { + return "android:"; + } + return String.Empty; + } + + float ComplexToFloat (uint value) + { + return (float)(value & 0xffffff00) * RadixMultipliers[(value >> 4) & 3]; + } + } + + bool SkipOverResourceMap (ARSCHeader header, BinaryReader reader) + { + Log.Debug ("AXML contains a resource map"); + + // Check size: < 8 bytes mean that the chunk is not complete + // Should be aligned to 4 bytes. + if (header.Size < 8 || (header.Size % 4) != 0) { + Log.Error ("Invalid chunk size in chunk XML_RESOURCE_MAP"); + return false; + } + + // Since our main interest is in reading AndroidManifest.xml, we're going to skip over the table + for (int i = 0; i < (header.Size - header.HeaderSize) / 4; i++) { + reader.ReadUInt32 (); + } + + return true; + } +} diff --git a/tools/apput/src/Android/AndroidManifest.cs b/tools/apput/src/Android/AndroidManifest.cs new file mode 100644 index 00000000000..c1778e528b3 --- /dev/null +++ b/tools/apput/src/Android/AndroidManifest.cs @@ -0,0 +1,58 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +public class AndroidManifest : IAspect +{ + public string Description { get; } + + AXMLParser? binaryParser; + + AndroidManifest (AXMLParser binaryParser, string? description) + { + Description = String.IsNullOrEmpty (description) ? "Android manifest" : description; + this.binaryParser = binaryParser; + } + + public static IAspect LoadAspect (Stream stream, IAspectState state, string? description) + { + var manifestState = state as AndroidManifestAspectState; + if (manifestState == null) { + throw new InvalidOperationException ("Internal error: unexpected aspect state. Was ProbeAspect unsuccessful?"); + } + + AndroidManifest ret; + if (manifestState.BinaryParser != null) { + ret = new AndroidManifest (manifestState.BinaryParser, description); + } else { + throw new NotImplementedException (); + } + ret.Read (); + + return ret; + } + + public static IAspectState ProbeAspect (Stream stream, string? description) + { + try { + stream.Seek (0, SeekOrigin.Begin); + + // The constructor will throw if it cannot recognize the format + var binaryParser = new AXMLParser (stream); + + // We leave parsing of the data to `LoadAspect`, here we only detect the format + return new AndroidManifestAspectState (binaryParser); + } catch (Exception ex) { + Log.Debug ($"Failed to instantiate AXML binary parser for '{description}'", ex); + } + + // TODO: detect plain XML + throw new NotImplementedException (); + } + + void Read () + { + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/Android/AndroidManifestAspectState.cs b/tools/apput/src/Android/AndroidManifestAspectState.cs new file mode 100644 index 00000000000..732e335c4a3 --- /dev/null +++ b/tools/apput/src/Android/AndroidManifestAspectState.cs @@ -0,0 +1,12 @@ +namespace ApplicationUtility; + +class AndroidManifestAspectState : IAspectState +{ + public bool Success => true; + public AXMLParser? BinaryParser { get; } + + public AndroidManifestAspectState (AXMLParser? binaryParser) + { + BinaryParser = binaryParser; + } +} diff --git a/tools/apput/src/Android/AndroidManifestAttributeType.cs b/tools/apput/src/Android/AndroidManifestAttributeType.cs new file mode 100644 index 00000000000..c8962929e36 --- /dev/null +++ b/tools/apput/src/Android/AndroidManifestAttributeType.cs @@ -0,0 +1,52 @@ +namespace ApplicationUtility; + +enum AndroidManifestAttributeType : uint +{ + // The 'data' field is either 0 or 1, specifying this resource is either undefined or empty, respectively. + Null = 0x00, + + // The 'data' field holds a ResTable_ref, a reference to another resource + Reference = 0x01, + + // The 'data' field holds an attribute resource identifier. + Attribute = 0x02, + + // The 'data' field holds an index into the containing resource table's global value string pool. + String = 0x03, + + // The 'data' field holds a single-precision floating point number. + Float = 0x04, + + // The 'data' holds a complex number encoding a dimension value such as "100in". + Dimension = 0x05, + + // The 'data' holds a complex number encoding a fraction of a container. + Fraction = 0x06, + + // The 'data' holds a dynamic ResTable_ref, which needs to be resolved before it can be used like a Reference + DynamicReference = 0x07, + + // The 'data' holds an attribute resource identifier, which needs to be resolved before it can be used like a Attribute. + DynamicAttribute = 0x08, + + // The 'data' is a raw integer value of the form n..n. + IntDec = 0x10, + + // The 'data' is a raw integer value of the form 0xn..n. + IntHex = 0x11, + + // The 'data' is either 0 or 1, for input "false" or "true" respectively. + IntBoolean = 0x12, + + // The 'data' is a raw integer value of the form #aarrggbb. + IntColorARGB8 = 0x1c, + + // The 'data' is a raw integer value of the form #rrggbb. + IntColorRGB8 = 0x1d, + + // The 'data' is a raw integer value of the form #argb. + IntColorARGB4 = 0x1e, + + // The 'data' is a raw integer value of the form #rgb. + IntColorRGB4 = 0x1f, +} diff --git a/tools/apput/src/Android/AndroidManifestChunkType.cs b/tools/apput/src/Android/AndroidManifestChunkType.cs new file mode 100644 index 00000000000..5b689bf3b1b --- /dev/null +++ b/tools/apput/src/Android/AndroidManifestChunkType.cs @@ -0,0 +1,23 @@ +namespace ApplicationUtility; + +enum AndroidManifestChunkType : ushort +{ + Null = 0x0000, + StringPool = 0x0001, + Table = 0x0002, + Xml = 0x0003, + + XmlFirstChunk = 0x0100, + XmlStartNamespace = 0x0100, + XmlEndNamespace = 0x0101, + XmlStartElement = 0x0102, + XmlEndElement = 0x0103, + XmlCData = 0x0104, + XmlLastChunk = 0x017f, + XmlResourceMap = 0x0180, + + TablePackage = 0x0200, + TableType = 0x0201, + TableTypeSpec = 0x0202, + TableLibrary = 0x0203, +} diff --git a/tools/apput/src/Android/AndroidManifestStringBlock.cs b/tools/apput/src/Android/AndroidManifestStringBlock.cs new file mode 100644 index 00000000000..884102de80e --- /dev/null +++ b/tools/apput/src/Android/AndroidManifestStringBlock.cs @@ -0,0 +1,176 @@ +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace ApplicationUtility; + +class AndroidManifestStringBlock +{ + const uint FlagSorted = 1 << 0; + const uint FlagUTF8 = 1 << 0; + + ARSCHeader header; + uint stringCount; + uint stringsOffset; + uint flags; + bool isUTF8; + List stringOffsets; + byte[] chars; + Dictionary stringCache; + + public uint StringCount => stringCount; + public bool IsUTF8 => isUTF8; + + public AndroidManifestStringBlock (Stream data, ARSCHeader stringPoolHeader) + { + header = stringPoolHeader; + + using var reader = Utilities.GetReaderAndRewindStream (data); + + stringCount = reader.ReadUInt32 (); + uint styleCount = reader.ReadUInt32 (); + + flags = reader.ReadUInt32 (); + isUTF8 = (flags & FlagUTF8) == FlagUTF8; + + stringsOffset = reader.ReadUInt32 (); + uint stylesOffset = reader.ReadUInt32 (); + + if (styleCount == 0 && stylesOffset > 0) { + Log.Info ("Styles Offset given, but styleCount is zero. This is not a problem but could indicate packers."); + } + + stringOffsets = new List (); + + for (uint i = 0; i < stringCount; i++) { + stringOffsets.Add (reader.ReadUInt32 ()); + } + + // We're not interested in styles, skip over their offsets + for (uint i = 0; i < styleCount; i++) { + reader.ReadUInt32 (); + } + + bool haveStyles = stylesOffset != 0 && styleCount != 0; + uint size = header.Size - stringsOffset; + if (haveStyles) { + size = stylesOffset - stringsOffset; + } + + if (size % 4 != 0) { + Log.Warning ("Size of strings is not aligned on four bytes."); + } + + chars = new byte[size]; + reader.Read (chars, 0, (int)size); + + if (haveStyles) { + size = header.Size - stylesOffset; + + if (size % 4 != 0) { + Log.Warning ("Size of styles is not aligned on four bytes."); + } + + // Not interested in them, skip + for (uint i = 0; i < size / 4; i++) { + reader.ReadUInt32 (); + } + } + + stringCache = new Dictionary (); + } + + public string? GetString (uint idx) + { + if (stringCache.TryGetValue (idx, out string? ret)) { + return ret; + } + + if (idx < 0 || idx > stringOffsets.Count || stringOffsets.Count == 0) { + return null; + } + + uint offset = stringOffsets[(int)idx]; + if (isUTF8) { + ret = DecodeUTF8 (offset); + } else { + ret = DecodeUTF16 (offset); + } + stringCache[idx] = ret; + + return ret; + } + + string DecodeUTF8 (uint offset) + { + // UTF-8 Strings contain two lengths, as they might differ: + // 1) the string length in characters + (uint length, uint nbytes) = DecodeLength (offset, sizeOfChar: 1); + offset += nbytes; + + // 2) the number of bytes the encoded string occupies + (uint encodedBytes, nbytes) = DecodeLength (offset, sizeOfChar: 1); + offset += nbytes; + + if (chars[offset + encodedBytes] != 0) { + throw new InvalidDataException ($"UTF-8 string is not NUL-terminated. Offset: offset"); + } + + return Encoding.UTF8.GetString (chars, (int)offset, (int)encodedBytes); + } + + string DecodeUTF16 (uint offset) + { + (uint length, uint nbytes) = DecodeLength (offset, sizeOfChar: 2); + offset += nbytes; + + uint encodedBytes = length * 2; + if (chars[offset + encodedBytes] != 0 && chars[offset + encodedBytes + 1] != 0) { + throw new InvalidDataException ($"UTF-16 string is not NUL-terminated. Offset: offset"); + } + + return Encoding.Unicode.GetString (chars, (int)offset, (int)encodedBytes); + } + + (uint length, uint nbytes) DecodeLength (uint offset, uint sizeOfChar) + { + uint sizeOfTwoChars = sizeOfChar << 1; + uint highBit = 0x80u << (8 * ((int)sizeOfChar - 1)); + uint length1, length2; + + // Length is tored as 1 or 2 characters of `sizeofChar` size + if (sizeOfChar == 1) { + // UTF-8 encoding, each character is a byte + length1 = chars[offset]; + length2 = chars[offset + 1]; + } else { + // UTF-16 encoding, each character is a short + length1 = (uint)((chars[offset]) | (chars[offset + 1] << 8)); + length2 = (uint)((chars[offset + 2]) | (chars[offset + 3] << 8)); + } + + uint length; + uint nbytes; + if ((length1 & highBit) != 0) { + length = ((length1 & ~highBit) << (8 * (int)sizeOfChar)) | length2; + nbytes = sizeOfTwoChars; + } else { + length = length1; + nbytes = sizeOfChar; + } + + // 8 bit strings: maximum of 0x7FFF bytes, http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#692 + // 16 bit strings: maximum of 0x7FFFFFF bytes, http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#670 + if (sizeOfChar == 1) { + if (length > 0x7fff) { + throw new InvalidDataException ("UTF-8 string is too long. Offset: {offset}"); + } + } else { + if (length > 0x7fffffff) { + throw new InvalidDataException ("UTF-16 string is too long. Offset: {offset}"); + } + } + + return (length, nbytes); + } +} diff --git a/tools/apput/src/Detector.cs b/tools/apput/src/Detector.cs index c9248e46e76..ea9ea5674e9 100644 --- a/tools/apput/src/Detector.cs +++ b/tools/apput/src/Detector.cs @@ -41,7 +41,7 @@ class Detector var flags = BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static; foreach (Type aspect in KnownTopLevelAspects) { - Log.Debug ($"Probing aspect: {aspect}"); + LogBanner ($"Probing aspect: {aspect}"); object? result = aspect.InvokeMember ( "ProbeAspect", flags, null, null, new object?[] { stream, description } @@ -52,7 +52,7 @@ class Detector continue; } - Log.Debug ($"Loading aspect: {aspect}"); + LogBanner ($"Loading aspect: {aspect}"); result = aspect.InvokeMember ( "LoadAspect", flags, null, null, new object?[] { stream, state, description } ); @@ -62,6 +62,14 @@ class Detector } return null; + + void LogBanner (string what) + { + Log.Debug (); + Log.Debug ("##########"); + Log.Debug (what); + Log.Debug (); + } } } diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index 0eb1461aa88..f3de6bdc38f 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -50,6 +50,8 @@ public abstract class ApplicationPackage : IAspect public List? AssemblyStores { get; protected set; } public List Architectures { get; protected set; } = new (); + AndroidManifest? manifest; + protected ApplicationPackage (ZipArchive zip, string? description) { Zip = zip; @@ -165,7 +167,7 @@ void TryLoadAssemblyStores () AssemblyStore? TryLoadAssemblyStore (string storePath) { // AssemblyStore class owns the stream, don't dispose it here - FileStream? storeStream = TryGetEntryStream (storePath); + Stream? storeStream = TryGetEntryStream (storePath); if (storeStream == null) { return null; } @@ -195,14 +197,29 @@ void TryLoadAndroidManifest () } Log.Debug ($"Found Android manifest '{AndroidManifestPath}'"); - using Stream? manifestStream = TryGetEntryStream (AndroidManifestPath); - // TODO: parse + + try { + Stream? manifestStream = TryGetEntryStream (AndroidManifestPath, extractToMemory: true); + if (manifestStream == null) { + Log.Error ("Failed to read android manifest from the application package."); + return; + } + IAspectState manifestState = AndroidManifest.ProbeAspect (manifestStream, AndroidManifestPath); + if (!manifestState.Success) { + Log.Debug ($"Failed to detect '{AndroidManifestPath}' package entry as supported Android manifest data."); + manifestStream.Dispose (); + return; + } + manifest = (AndroidManifest)AndroidManifest.LoadAspect (manifestStream, manifestState, AndroidManifestPath); + } catch (Exception ex) { + Log.Debug ($"Failed to load android manifest '{AndroidManifestPath}' from the archive.", ex); + } } string GetNativeLibDir (AndroidTargetArch arch) => $"{NativeLibDirBase}/{MonoAndroidHelper.ArchToAbi (arch)}/"; string GetNativeLibFile (AndroidTargetArch arch, string fileName) => $"{GetNativeLibDir (arch)}{fileName}"; - FileStream? TryGetEntryStream (string path) + Stream? TryGetEntryStream (string path, bool extractToMemory = false) { try { ZipArchiveEntry? entry = Zip.GetEntry (path); @@ -211,6 +228,15 @@ void TryLoadAndroidManifest () return null; } + if (extractToMemory) { + Log.Debug ($"Extracting entry '{path}' to a memory stream"); + using var inputStream = entry.Open (); + var outputStream = new MemoryStream (); + inputStream.CopyTo (outputStream); + inputStream.Flush (); + return outputStream; + } + string tempFile = Path.GetTempFileName (); TempFileManager.RegisterFile (tempFile); @@ -219,8 +245,6 @@ void TryLoadAndroidManifest () return File.OpenRead (tempFile); } catch (Exception ex) { Log.Debug ($"Failed to load entry '{path}' from the archive.", ex); - - // TODO: remove temp file (using a helper method, which doesn't exist yet) return null; } } From ac8ee2258aae4bc19b00c58aeef6b03c440c0976 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 15 Jul 2025 16:41:04 +0200 Subject: [PATCH 12/12] Android manifest almost finished (protobuf missing) + native bits Beginnings of support for reading ELF binaries, needed for typemaps and friends --- tools/apput/src/Android/ARSCHeader.cs | 2 +- tools/apput/src/Android/AXMLParser.cs | 2 +- tools/apput/src/Android/AndroidManifest.cs | 44 ++- .../src/Android/AndroidManifestAspectState.cs | 8 + tools/apput/src/Common/Utilities.cs | 9 +- tools/apput/src/Native/AnELF.cs | 331 ++++++++++++++++++ tools/apput/src/Native/ELF32.cs | 91 +++++ tools/apput/src/Native/ELF64.cs | 206 +++++++++++ .../src/Native/ELF_RelocationWithAddend.cs | 52 +++ tools/apput/src/Native/LibXamarinApp.cs | 38 ++ tools/apput/src/Native/NativeAppInfo.cs | 8 + tools/apput/src/Native/NativeUtils.cs | 72 ++++ tools/apput/src/Native/RelocationSection.cs | 30 ++ tools/apput/src/Native/RelocationTypes.cs | 118 +++++++ tools/apput/src/Native/SharedLibrary.cs | 6 +- tools/apput/src/Package/ApplicationPackage.cs | 37 ++ 16 files changed, 1043 insertions(+), 11 deletions(-) create mode 100644 tools/apput/src/Native/AnELF.cs create mode 100644 tools/apput/src/Native/ELF32.cs create mode 100644 tools/apput/src/Native/ELF64.cs create mode 100644 tools/apput/src/Native/ELF_RelocationWithAddend.cs create mode 100644 tools/apput/src/Native/LibXamarinApp.cs create mode 100644 tools/apput/src/Native/NativeAppInfo.cs create mode 100644 tools/apput/src/Native/NativeUtils.cs create mode 100644 tools/apput/src/Native/RelocationSection.cs create mode 100644 tools/apput/src/Native/RelocationTypes.cs diff --git a/tools/apput/src/Android/ARSCHeader.cs b/tools/apput/src/Android/ARSCHeader.cs index 446d831ad8a..3872a0a6446 100644 --- a/tools/apput/src/Android/ARSCHeader.cs +++ b/tools/apput/src/Android/ARSCHeader.cs @@ -28,7 +28,7 @@ public ARSCHeader (Stream data, AndroidManifestChunkType? expectedType = null) } // Data in AXML is little-endian, which is fortuitous as that's the only format BinaryReader understands. - using BinaryReader reader = Utilities.GetReaderAndRewindStream (data); + using BinaryReader reader = Utilities.GetReaderAndRewindStream (data, rewindStream: false); // ushort: type // ushort: header_size diff --git a/tools/apput/src/Android/AXMLParser.cs b/tools/apput/src/Android/AXMLParser.cs index 80c9151c6aa..6df7ecd7a6d 100644 --- a/tools/apput/src/Android/AXMLParser.cs +++ b/tools/apput/src/Android/AXMLParser.cs @@ -108,7 +108,7 @@ public AXMLParser (Stream data) { valid = true; - XmlDocument ret = new XmlDocument (); + var ret = new XmlDocument (); XmlDeclaration declaration = ret.CreateXmlDeclaration ("1.0", stringPool.IsUTF8 ? "UTF-8" : "UTF-16", null); ret.InsertBefore (declaration, ret.DocumentElement); diff --git a/tools/apput/src/Android/AndroidManifest.cs b/tools/apput/src/Android/AndroidManifest.cs index c1778e528b3..e709c053641 100644 --- a/tools/apput/src/Android/AndroidManifest.cs +++ b/tools/apput/src/Android/AndroidManifest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Xml; namespace ApplicationUtility; @@ -8,6 +9,7 @@ public class AndroidManifest : IAspect public string Description { get; } AXMLParser? binaryParser; + XmlDocument? xmlDoc; AndroidManifest (AXMLParser binaryParser, string? description) { @@ -35,6 +37,7 @@ public static IAspect LoadAspect (Stream stream, IAspectState state, string? des public static IAspectState ProbeAspect (Stream stream, string? description) { + Log.Debug ($"Checking if '{description}' is an Android binary XML document."); try { stream.Seek (0, SeekOrigin.Begin); @@ -44,15 +47,48 @@ public static IAspectState ProbeAspect (Stream stream, string? description) // We leave parsing of the data to `LoadAspect`, here we only detect the format return new AndroidManifestAspectState (binaryParser); } catch (Exception ex) { - Log.Debug ($"Failed to instantiate AXML binary parser for '{description}'", ex); + Log.Debug ($"Failed to instantiate AXML binary parser for '{description}'. Exception thrown:", ex); } - // TODO: detect plain XML - throw new NotImplementedException (); + Log.Debug ($"Checking if '{description}' is an plain XML document."); + try { + return new AndroidManifestAspectState (ParsePlainXML (stream)); + } catch (Exception ex) { + Log.Debug ($"Failed to parse '{description}' as XML document. Exception thrown:", ex); + } + + // TODO: AndroidManifest.xml in AAB files is actually a protobuf data dump. Attempt to + // deserialize it here. + return new BasicAspectState (success: false); } void Read () { - throw new NotImplementedException (); + if (binaryParser == null) { + throw new NotImplementedException (); + } + + xmlDoc = binaryParser.Parse (); + if (xmlDoc == null || !binaryParser.IsValid) { + Log.Debug ($"AXML parser didn't render a valid document for '{Description}'"); + return; + } + Log.Debug ($"'{Description}' loaded and parsed correctly."); + } + + static XmlDocument ParsePlainXML (Stream stream) + { + stream.Seek (0, SeekOrigin.Begin); + var settings = new XmlReaderSettings { + IgnoreComments = true, + IgnoreProcessingInstructions = true, + IgnoreWhitespace = true, + }; + + using var reader = XmlReader.Create (stream, settings); + var doc = new XmlDocument (); + doc.Load (reader); + + return doc; } } diff --git a/tools/apput/src/Android/AndroidManifestAspectState.cs b/tools/apput/src/Android/AndroidManifestAspectState.cs index 732e335c4a3..863b63ef3af 100644 --- a/tools/apput/src/Android/AndroidManifestAspectState.cs +++ b/tools/apput/src/Android/AndroidManifestAspectState.cs @@ -1,12 +1,20 @@ +using System.Xml; + namespace ApplicationUtility; class AndroidManifestAspectState : IAspectState { public bool Success => true; public AXMLParser? BinaryParser { get; } + public XmlDocument? Xml { get; } public AndroidManifestAspectState (AXMLParser? binaryParser) { BinaryParser = binaryParser; } + + public AndroidManifestAspectState (XmlDocument? xmlDoc) + { + Xml = xmlDoc; + } } diff --git a/tools/apput/src/Common/Utilities.cs b/tools/apput/src/Common/Utilities.cs index b2f51ca132e..a5ec008cc04 100644 --- a/tools/apput/src/Common/Utilities.cs +++ b/tools/apput/src/Common/Utilities.cs @@ -33,9 +33,12 @@ public static void CloseAndDeleteFile (FileStream stream, bool quiet = true) DeleteFile (path); } - public static BinaryReader GetReaderAndRewindStream (Stream stream) + public static BinaryReader GetReaderAndRewindStream (Stream stream, bool rewindStream = false) { - stream.Seek (0, SeekOrigin.Begin); + if (rewindStream) { + stream.Seek (0, SeekOrigin.Begin); + } + return new BinaryReader (stream, Encoding.UTF8, leaveOpen: true); } @@ -44,4 +47,6 @@ public static BasicAspectState GetFailureAspectState (string message) Log.Debug (message); return new BasicAspectState (false); } + + public static string ToStringOrNull (T? reference) => reference == null ? "" : reference.ToString () ?? "[unknown]"; } diff --git a/tools/apput/src/Native/AnELF.cs b/tools/apput/src/Native/AnELF.cs new file mode 100644 index 00000000000..5dfec85c2fb --- /dev/null +++ b/tools/apput/src/Native/AnELF.cs @@ -0,0 +1,331 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; + +using ELFSharp; +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace ApplicationUtility; + +abstract class AnELF +{ + protected static readonly byte[] EmptyArray = Array.Empty (); + + const string DynsymSectionName = ".dynsym"; + const string SymtabSectionName = ".symtab"; + const string RodataSectionName = ".rodata"; + + ISymbolTable dynamicSymbolsSection; + ISection rodataSection; + ISymbolTable? symbolsSection; + string filePath; + IELF elf; + Stream elfStream; + + protected ISymbolTable DynSymSection => dynamicSymbolsSection; + protected ISymbolTable? SymSection => symbolsSection; + protected ISection RodataSection => rodataSection; + public IELF AnyELF => elf; + protected Stream ELFStream => elfStream; + + public string FilePath => filePath; + public int PointerSize => Is64Bit ? 8 : 4; + + public abstract bool Is64Bit { get; } + public abstract string Bitness { get; } + + protected AnELF (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + { + this.filePath = filePath; + this.elf = elf; + elfStream = stream; + dynamicSymbolsSection = dynsymSection; + this.rodataSection = rodataSection; + symbolsSection = symSection; + } + + public ISymbolEntry? GetSymbol (string symbolName) + { + ISymbolEntry? symbol = null; + + if (symbolsSection != null) { + symbol = GetSymbol (symbolsSection, symbolName); + } + + if (symbol == null) { + symbol = GetSymbol (dynamicSymbolsSection, symbolName); + } + + return symbol; + } + + protected static ISymbolEntry? GetSymbol (ISymbolTable symtab, string symbolName) + { + return symtab.Entries.Where (entry => String.Compare (entry.Name, symbolName, StringComparison.Ordinal) == 0).FirstOrDefault (); + } + + protected static SymbolEntry? GetSymbol (SymbolTable symtab, T symbolValue) where T: struct + { + return symtab.Entries.Where (entry => entry.Value.Equals (symbolValue)).FirstOrDefault (); + } + + public bool HasSymbol (string symbolName) + { + return GetSymbol (symbolName) != null; + } + + public byte[] GetData (string symbolName) + { + return GetData (symbolName, out ISymbolEntry? _); + } + + public byte[] GetData (string symbolName, out ISymbolEntry? symbolEntry) + { + Log.Debug ($"Looking for symbol: {symbolName}"); + symbolEntry = GetSymbol (symbolName); + if (symbolEntry == null) + return EmptyArray; + + if (Is64Bit) { + var symbol64 = symbolEntry as SymbolEntry; + if (symbol64 == null) + throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 64-bit symbol"); + return GetData (symbol64); + } + + var symbol32 = symbolEntry as SymbolEntry; + if (symbol32 == null) + throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 32-bit symbol"); + + return GetData (symbol32); + } + + public string? GetStringFromPointer (ISymbolEntry symbolEntry) + { + return GetStringFromPointerField (symbolEntry, 0); + } + + public abstract string? GetStringFromPointerField (ISymbolEntry symbolEntry, ulong pointerFieldOffset); + public abstract byte[] GetData (ulong symbolValue, ulong size); + + public string? GetASCIIZ (ulong symbolValue) + { + return GetASCIIZ (GetData (symbolValue, 0), 0); + } + + public string? GetASCIIZ (byte[] data, ulong offset) + { + if (offset >= (ulong)data.LongLength) { + Log.Debug ("Not enough data to retrieve an ASCIIZ string"); + return null; + } + + int count = data.Length; + + for (ulong i = offset; i < (ulong)data.LongLength; i++) { + if (data[i] == 0) { + count = (int)(i - offset); + break; + } + } + + return Encoding.ASCII.GetString (data, (int)offset, count); + } + + public ulong GetPaddedSize (ulong sizeSoFar) => NativeUtils.GetPaddedSize (sizeSoFar, Is64Bit); + + public ulong GetPaddedSize (ulong sizeSoFar, S _) + { + return GetPaddedSize (sizeSoFar); + } + + protected virtual byte[] GetData (SymbolEntry symbol) + { + throw new NotSupportedException (); + } + + protected virtual byte[] GetData (SymbolEntry symbol) + { + throw new NotSupportedException (); + } + + protected byte[] GetData (ISymbolEntry symbol, ulong size, ulong offset) + { + return GetData (symbol.PointedSection, size, offset); + } + + protected byte[] GetData (ISection section, ulong size, ulong offset) + { + ulong sectionOffset = (elf.Class == Class.Bit64 ? ((Section)section).Offset : ((Section)section).Offset); + Log.Debug ($"AnELF.GetData: section == {section.Name}; type == {section.Type}; flags == {section.Flags}; offset into binary == {sectionOffset}; size == {size}"); + byte[] data = section.GetContents (); + + Log.Debug ($" section data length: {data.Length} (long: {data.LongLength})"); + Log.Debug ($" offset into section: {offset}; symbol data length: {size}"); + if ((ulong)data.LongLength < (offset + size)) { + return EmptyArray; + } + + if (size == 0) + size = (ulong)data.Length - offset; + + var ret = new byte[size]; + checked { + Array.Copy (data, (int)offset, ret, 0, (int)size); + } + + return ret; + } + + public uint GetUInt32 (string symbolName) + { + return GetUInt32 (GetData (symbolName), 0, symbolName); + } + + public uint GetUInt32 (ulong symbolValue) + { + return GetUInt32 (GetData (symbolValue, 4), 0, symbolValue.ToString ()); + } + + protected uint GetUInt32 (byte[] data, ulong offset, string symbolName) + { + if (data.Length < 4) { + throw new InvalidOperationException ($"Data not big enough to retrieve a 32-bit integer from it (need 4, got {data.Length})"); + } + + return BitConverter.ToUInt32 (GetIntegerData (4, data, offset, symbolName), 0); + } + + public ulong GetUInt64 (string symbolName) + { + return GetUInt64 (GetData (symbolName), 0, symbolName); + } + + public ulong GetUInt64 (ulong symbolValue) + { + return GetUInt64 (GetData (symbolValue, 8), 0, symbolValue.ToString ()); + } + + protected ulong GetUInt64 (byte[] data, ulong offset, string symbolName) + { + if (data.Length < 8) { + throw new InvalidOperationException ("Data not big enough to retrieve a 64-bit integer from it"); + } + return BitConverter.ToUInt64 (GetIntegerData (8, data, offset, symbolName), 0); + } + + byte[] GetIntegerData (uint size, byte[] data, ulong offset, string symbolName) + { + if ((ulong)data.LongLength < (offset + size)) { + string bits = size == 4 ? "32" : "64"; + throw new InvalidOperationException ($"Unable to read UInt{bits} value for symbol '{symbolName}': data not long enough"); + } + + byte[] ret = new byte[size]; + Array.Copy (data, (int)offset, ret, 0, ret.Length); + Endianess myEndianness = BitConverter.IsLittleEndian ? Endianess.LittleEndian : Endianess.BigEndian; + if (AnyELF.Endianess != myEndianness) { + Array.Reverse (ret); + } + + return ret; + } + + public static bool TryLoad (string filePath, out AnELF? anElf) + { + using var fs = File.OpenRead (filePath); + return TryLoad (fs, filePath, out anElf); + } + + public static bool TryLoad (Stream stream, string filePath, out AnELF? anElf) + { + anElf = null; + Class elfClass = ELFReader.CheckELFType (stream); + if (elfClass == Class.NotELF) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not an ELF binary"); + return false; + } + + IELF elf = ELFReader.Load (stream, shouldOwnStream: false); + + if (elf.Type != FileType.SharedObject) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not a shared library"); + return false; + } + + if (elf.Endianess != Endianess.LittleEndian) { + Log.Warning ($"AnELF.TryLoad: {filePath} is not a little-endian binary"); + return false; + } + + bool is64; + switch (elf.Machine) { + case Machine.ARM: + case Machine.Intel386: + is64 = false; + + break; + + case Machine.AArch64: + case Machine.AMD64: + is64 = true; + + break; + + default: + Log.Warning ($"{filePath} is for an unsupported machine type {elf.Machine}"); + return false; + } + + ISymbolTable? symtab = GetSymbolTable (elf, DynsymSectionName); + if (symtab == null) { + Log.Warning ($"{filePath} does not contain dynamic symbol section '{DynsymSectionName}'"); + return false; + } + ISymbolTable dynsym = symtab; + + ISection? sec = GetSection (elf, RodataSectionName); + if (sec == null) { + Log.Warning ("${filePath} does not contain read-only data section ('{RodataSectionName}')"); + return false; + } + ISection rodata = sec; + + ISymbolTable? sym = GetSymbolTable (elf, SymtabSectionName); + + if (is64) { + anElf = new ELF64 (stream, filePath, elf, dynsym, rodata, sym); + } else { + anElf = new ELF32 (stream, filePath, elf, dynsym, rodata, sym); + } + + Log.Debug ($"AnELF.TryLoad: {filePath} is a {anElf.Bitness}-bit ELF binary ({elf.Machine})"); + return true; + } + + protected static ISymbolTable? GetSymbolTable (IELF elf, string sectionName) + { + ISection? section = GetSection (elf, sectionName); + if (section == null) { + return null; + } + + var symtab = section as ISymbolTable; + if (symtab == null) { + return null; + } + + return symtab; + } + + protected static ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } +} diff --git a/tools/apput/src/Native/ELF32.cs b/tools/apput/src/Native/ELF32.cs new file mode 100644 index 00000000000..03a0d921f52 --- /dev/null +++ b/tools/apput/src/Native/ELF32.cs @@ -0,0 +1,91 @@ +using System; +using System.IO; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace ApplicationUtility; + +class ELF32 : AnELF +{ + public override bool Is64Bit => false; + public override string Bitness => "32"; + + SymbolTable DynamicSymbols => (SymbolTable)DynSymSection; + SymbolTable? Symbols => (SymbolTable?)SymSection; + Section Rodata => (Section)RodataSection; + ELF ELF => (ELF)AnyELF; + + public ELF32 (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + : base (stream, filePath, elf, dynsymSection, rodataSection, symSection) + {} + + public override string GetStringFromPointerField(ISymbolEntry symbolEntry, ulong pointerFieldOffset) + { + throw new NotImplementedException(); + } + + public override byte[] GetData (ulong symbolValue, ulong size = 0) + { + checked { + return GetData ((uint)symbolValue, size); + } + } + + byte[] GetData (uint symbolValue, ulong size) + { + Log.Debug ($"ELF64.GetData: Looking for symbol value {symbolValue:X08}"); + + SymbolEntry? symbol = GetSymbol (DynamicSymbols, symbolValue); + if (symbol == null && Symbols != null) { + symbol = GetSymbol (Symbols, symbolValue); + } + + if (symbol != null) { + Log.Debug ($"ELF64.GetData: found in section {symbol.PointedSection.Name}"); + return GetData (symbol); + } + + Section section = FindSectionForValue (symbolValue); + + Log.Debug ($"ELF64.GetData: found in section {section} {section.Name}"); + return GetData (section, size, OffsetInSection (section, symbolValue)); + } + + protected override byte[] GetData (SymbolEntry symbol) + { + ulong offset = symbol.Value - symbol.PointedSection.LoadAddress; + return GetData (symbol, symbol.Size, offset); + } + + Section FindSectionForValue (uint symbolValue) + { + Log.Debug ($"FindSectionForValue ({symbolValue:X08})"); + int nsections = ELF.Sections.Count; + + for (int i = nsections - 1; i >= 0; i--) { + Section section = ELF.GetSection (i); + if (section.Type != SectionType.ProgBits) + continue; + + if (SectionInRange (section, symbolValue)) + return section; + } + + throw new InvalidOperationException ($"Section matching symbol value {symbolValue:X08} cannot be found"); + } + + bool SectionInRange (Section section, uint symbolValue) + { + Log.Debug ($"SectionInRange ({section.Name}, {symbolValue:X08})"); + Log.Debug ($" address == {section.LoadAddress:X08}; size == {section.Size}; last address = {section.LoadAddress + section.Size:X08}"); + Log.Debug ($" symbolValue >= section.LoadAddress? {symbolValue >= section.LoadAddress}"); + Log.Debug ($" (section.LoadAddress + section.Size) >= symbolValue? {(section.LoadAddress + section.Size) >= symbolValue}"); + return symbolValue >= section.LoadAddress && (section.LoadAddress + section.Size) >= symbolValue; + } + + ulong OffsetInSection (Section section, uint symbolValue) + { + return symbolValue - section.LoadAddress; + } +} diff --git a/tools/apput/src/Native/ELF64.cs b/tools/apput/src/Native/ELF64.cs new file mode 100644 index 00000000000..67ba5193c71 --- /dev/null +++ b/tools/apput/src/Native/ELF64.cs @@ -0,0 +1,206 @@ +using System; +using System.IO; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace ApplicationUtility; + +class ELF64 : AnELF +{ + public override bool Is64Bit => true; + public override string Bitness => "64"; + + SymbolTable DynamicSymbols => (SymbolTable)DynSymSection; + SymbolTable? Symbols => (SymbolTable?)SymSection; + Section Rodata => (Section)RodataSection; + ELF ELF => (ELF)AnyELF; + + public ELF64 (Stream stream, string filePath, IELF elf, ISymbolTable dynsymSection, ISection rodataSection, ISymbolTable? symSection) + : base (stream, filePath, elf, dynsymSection, rodataSection, symSection) + {} + + public override string? GetStringFromPointerField (ISymbolEntry symbolEntry, ulong pointerFieldOffset) + { + var symbol = symbolEntry as SymbolEntry; + if (symbol == null) { + throw new InvalidOperationException ($"Expected a 64-bit symbol entry, got {symbolEntry}"); + } + + switch (ELF.Machine) { + case Machine.AArch64: + return GetStringFromPointerField_ARM64 (symbol, pointerFieldOffset); + + case Machine.AMD64: + return GetStringFromPointerField_X64 (symbol, pointerFieldOffset); + + default: + throw new InvalidOperationException ($"Unsupported ELF machine type '{ELF.Machine}'"); + } + } + + string? GetStringFromPointerField_ARM64 (SymbolEntry symbolEntry, ulong pointerFieldOffset) + { + return GetStringFromPointerField_Common ( + symbolEntry, + pointerFieldOffset, + (ELF64_Rela rela) => { + // We only support R_AARCH64_RELATIVE right now + return (RelocationTypeARM64)rela.r_info == RelocationTypeARM64.R_AARCH64_RELATIVE; + } + ); + } + + string? GetStringFromPointerField_X64 (SymbolEntry symbolEntry, ulong pointerFieldOffset) + { + return GetStringFromPointerField_Common ( + symbolEntry, + pointerFieldOffset, + (ELF64_Rela rela) => { + // We only support R_X86_64_RELATIVE right now + return (RelocationTypeX64)rela.r_info == RelocationTypeX64.R_X86_64_RELATIVE; + } + ); + } + + string? GetStringFromPointerField_Common (SymbolEntry symbolEntry, ulong pointerFieldOffset, Func validRelocation) + { + Log.Debug ($"[ARM64] Getting string from a pointer field in symbol '{symbolEntry.Name}', at offset {pointerFieldOffset} into the structure"); + + if (symbolEntry.PointedSection.Type != SectionType.ProgBits || !symbolEntry.PointedSection.Flags.HasFlag (SectionFlags.Writable)) { + Log.Debug (" Symbol section isn't a writable data one, pointers require a writable section to apply relocations"); + Log.Debug ($" Section info: {symbolEntry.PointedSection}"); + return null; + } + + // Steps: + // + // 1. Calculate address of the field in the symbol data: [symbol section virtual address] + [symbol offset into section] + pointerFieldOffset + // ELFSharp does part of the job for us - symbol's value is its virtual address + ulong pointerVA = symbolEntry.Value + pointerFieldOffset; + Log.Debug ($" Section address == 0x{symbolEntry.PointedSection.LoadAddress:x}; offset == 0x{symbolEntry.PointedSection.Offset:x}"); + Log.Debug ($" Symbol entry value == 0x{symbolEntry.Value:x}"); + Log.Debug ($" Virtual address of the pointer: 0x{pointerVA:x} ({pointerVA})"); + + // 2. Find the .rela.dyn section + const string RelaDynSectionName = ".rela.dyn"; + Section? relaDynSection = ELF.GetSection (RelaDynSectionName); + Log.Debug ($" Relocation section: {Utilities.ToStringOrNull (relaDynSection)}"); + if (relaDynSection == null) { + Log.Debug ($" Section '{RelaDynSectionName}' not found"); + return null; + } + + // Make sure section type is what we need and expect + if (relaDynSection.Type != SectionType.RelocationAddends) { + Log.Debug ($" Section '{RelaDynSectionName}' has invalid type. Expected {SectionType.RelocationAddends}, got {relaDynSection.Type}"); + return null; + } + var relocationReader = new RelocationSectionAddend64 (relaDynSection); + + // 3. Find relocation entry with offset matching the address calculated in 1. Relocation entry should have code 0x403 (1027) - R_AARCH64_RELATIVE + if (!relocationReader.Entries.TryGetValue (pointerVA, out ELF64_Rela? relocation) || relocation == null) { + Log.Debug ($" Relocation for pointer address 0x{pointerVA:x} not found"); + return null; + } + Log.Debug ($" Found relocation: {relocation}"); + + if (!validRelocation (relocation)) { + // Yell, so that we can fix it + throw new NotSupportedException ($"AArch64 relocation type {relocation.r_info} not supported. Please report at https://github.com/xamarin/xamarin.android/issues/"); + } + + // 4. Read relocation entry (see elf(5) for Elf32_Rela and Elf64_Rela structures) and get the addend value + ulong addend = (ulong)relocation.r_addend; + + // 5. Find section the addend from 4. falls within + Section? pointeeSection = FindSectionForValue (addend); + if (pointeeSection == null) { + Log.Debug ($" Unable to find section in which pointee 0x{addend:x} resides"); + return null; + } + Log.Debug ($" Pointee 0x{addend:x} falls within section {pointeeSection}"); + + // 6. Read that section data + byte[] data = pointeeSection.GetContents (); + + // 7. Subtract section address from the addend, this will give offset into the section + ulong addendSectionOffset = addend - pointeeSection.LoadAddress; + Log.Debug ($" Pointee offset into section data == 0x{addendSectionOffset:x} ({addendSectionOffset})"); + + // 8. Read ASCIIZ data from the offset obtained in 7. + return GetASCIIZ (data, addendSectionOffset); + } + + public override byte[] GetData (ulong symbolValue, ulong size = 0) + { + Log.Debug ($"ELF64.GetData: Looking for symbol value {symbolValue:X08}"); + + SymbolEntry? symbol = GetSymbol (DynamicSymbols, symbolValue); + if (symbol == null && Symbols != null) { + symbol = GetSymbol (Symbols, symbolValue); + } + + if (symbol != null) { + Log.Debug ($"ELF64.GetData: found in section {symbol.PointedSection.Name}"); + if (symbol.Size == 0) { + return EmptyArray; + } + + return GetData (symbol); + } + + Section section = FindProgBitsSectionForValue (symbolValue); + + Log.Debug ($"ELF64.GetData: found in section {section} {section.Name}"); + return GetData (section, size, OffsetInSection (section, symbolValue)); + } + + protected override byte[] GetData (SymbolEntry symbol) + { + if (symbol.Size == 0) { + return EmptyArray; + } + + return GetData (symbol, symbol.Size, OffsetInSection (symbol.PointedSection, symbol.Value)); + } + + Section FindProgBitsSectionForValue (ulong symbolValue) + { + return FindSectionForValue (symbolValue, SectionType.ProgBits) ?? throw new InvalidOperationException ($"Section matching symbol value {symbolValue:X08} cannot be found"); + } + + Section? FindSectionForValue (ulong symbolValue, SectionType requiredType = SectionType.Null) + { + Log.Debug ($"FindSectionForValue ({symbolValue:X08}, {requiredType})"); + int nsections = ELF.Sections.Count; + + for (int i = nsections - 1; i >= 0; i--) { + Section section = ELF.GetSection (i); + if (requiredType != SectionType.Null && section.Type != requiredType) { + continue; + } + + if (SectionInRange (section, symbolValue)) { + return section; + } + } + + Log.Debug ($"Section matching symbol value {symbolValue:X08} cannot be found"); + return null; + } + + bool SectionInRange (Section section, ulong symbolValue) + { + Log.Debug ($"SectionInRange ({section.Name}, {symbolValue:X08})"); + Log.Debug ($" address == {section.LoadAddress:X08}; size == {section.Size}; last address = {section.LoadAddress + section.Size:X08}"); + Log.Debug ($" symbolValue >= section.LoadAddress? {symbolValue >= section.LoadAddress}"); + Log.Debug ($" (section.LoadAddress + section.Size) >= symbolValue? {(section.LoadAddress + section.Size) >= symbolValue}"); + return symbolValue >= section.LoadAddress && (section.LoadAddress + section.Size) >= symbolValue; + } + + ulong OffsetInSection (Section section, ulong symbolValue) + { + return symbolValue - section.LoadAddress; + } +} diff --git a/tools/apput/src/Native/ELF_RelocationWithAddend.cs b/tools/apput/src/Native/ELF_RelocationWithAddend.cs new file mode 100644 index 00000000000..d50c28a2d69 --- /dev/null +++ b/tools/apput/src/Native/ELF_RelocationWithAddend.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +abstract class ELF_Rela +{ + protected abstract long StructureSize { get; } + + public readonly TUnsigned r_offset; + public readonly TUnsigned r_info; + public readonly TSigned r_addend; + + protected ELF_Rela (BinaryReader reader, ulong offsetIntoData) + { + if (reader.BaseStream.Length < (long)offsetIntoData + StructureSize) { + throw new ArgumentOutOfRangeException ("Data array too short"); + } + ReadData (reader, out r_offset, out r_info, out r_addend); + } + + protected abstract void ReadData (BinaryReader reader, out TUnsigned offset, out TUnsigned info, out TSigned addend); + + public override string ToString() + { + return $"{GetType ()}: r_offset == 0x{r_offset:x} ({r_offset}); r_info == 0x{r_info:x} ({r_info}); r_addend == 0x{r_addend:x} ({r_addend})"; + } +} + +// Corresponds to Elf64_Rela structure from ELF documentation: +// +// typedef struct { +// Elf64_Addr r_offset; +// uint64_t r_info; +// int64_t r_addend; +// } Elf64_Rela; +// +sealed class ELF64_Rela : ELF_Rela +{ + protected override long StructureSize => 3 * sizeof (ulong); + + public ELF64_Rela (BinaryReader data, ulong offsetIntoData) + : base (data, offsetIntoData) + {} + + protected override void ReadData (BinaryReader reader, out ulong offset, out ulong info, out long addend) + { + offset = reader.ReadUInt64 (); + info = reader.ReadUInt64 (); + addend = reader.ReadInt64 (); + } +} diff --git a/tools/apput/src/Native/LibXamarinApp.cs b/tools/apput/src/Native/LibXamarinApp.cs new file mode 100644 index 00000000000..572d07d4ce2 --- /dev/null +++ b/tools/apput/src/Native/LibXamarinApp.cs @@ -0,0 +1,38 @@ +using System; +using System.IO; + +namespace ApplicationUtility; + +// TODO: make it an abstract class, we need to support different formats +class LibXamarinApp : SharedLibrary, IAspect +{ + LibXamarinApp (Stream stream, string description) + : base (stream, description) + {} + + public static new IAspect LoadAspect (Stream stream, IAspectState state, string? description) + { + if (String.IsNullOrEmpty (description)) { + throw new ArgumentException ("Must be a shared library name", nameof (description)); + } + + if (!IsSupportedELFSharedLibrary (stream, description)) { + throw new InvalidOperationException ("Stream is not a supported ELF shared library"); + } + + // TODO: this needs to be versioned + return new LibXamarinApp (stream, description); + } + + public static new IAspectState ProbeAspect (Stream stream, string? description) + { + IAspectState sharedLibState = SharedLibrary.ProbeAspect (stream, description); + if (!sharedLibState.Success) { + return sharedLibState; + } + + // TODO: check for presence of a handful of fields and read at least `format_tag` to determine + // format version. + throw new NotImplementedException (); + } +} diff --git a/tools/apput/src/Native/NativeAppInfo.cs b/tools/apput/src/Native/NativeAppInfo.cs new file mode 100644 index 00000000000..7f026d8ecc5 --- /dev/null +++ b/tools/apput/src/Native/NativeAppInfo.cs @@ -0,0 +1,8 @@ +namespace ApplicationUtility; + +public class NativeAppInfo +{ + internal NativeAppInfo (LibXamarinApp xamarinAppLibrary) + { + } +} diff --git a/tools/apput/src/Native/NativeUtils.cs b/tools/apput/src/Native/NativeUtils.cs new file mode 100644 index 00000000000..4c0648b0345 --- /dev/null +++ b/tools/apput/src/Native/NativeUtils.cs @@ -0,0 +1,72 @@ +using System; + +namespace ApplicationUtility; + +class NativeUtils +{ + static ulong GetPadding (ulong sizeSoFar, bool is64Bit, out ulong typeSize) + { + typeSize = GetNativeTypeSize (is64Bit); + if (typeSize == 1) { + return 0; + } + + ulong modulo; + if (is64Bit) { + modulo = typeSize < 8 ? 4u : 8u; + } else { + modulo = 4u; + } + + ulong alignment = sizeSoFar % modulo; + if (alignment == 0) { + return 0; + } + + return modulo - alignment; + } + + public static ulong GetPadding (ulong sizeSoFar, bool is64Bit) + { + return GetPadding (sizeSoFar, is64Bit, out ulong _); + } + + public static ulong GetPaddedSize (ulong sizeSoFar, bool is64Bit) + { + ulong padding = GetPadding (sizeSoFar, is64Bit, out ulong typeSize); + + if (padding == 0) { + return typeSize; + } + + return typeSize + padding; + } + + public static ulong GetNativeTypeSize (bool is64Bit) + { + Type type = typeof(S); + + if (type == typeof(string) || type == typeof(IntPtr)) { + // We treat `string` as a generic pointer + return is64Bit ? 8u : 4u; + } + + if (type == typeof(byte)) { + return 1u; + } + + if (type == typeof(bool)) { + return 1u; + } + + if (type == typeof(Int32) || type == typeof(UInt32)) { + return 4u; + } + + if (type == typeof(Int64) || type == typeof(UInt64)) { + return 8u; + } + + throw new InvalidOperationException ($"Unable to map managed type {type} to native assembler type"); + } +} diff --git a/tools/apput/src/Native/RelocationSection.cs b/tools/apput/src/Native/RelocationSection.cs new file mode 100644 index 00000000000..f496639057d --- /dev/null +++ b/tools/apput/src/Native/RelocationSection.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using System.IO; + +using ELFSharp.ELF.Sections; + +namespace ApplicationUtility; + +abstract class RelocationSectionAddend + where TUnsigned: notnull + where TRela: notnull, ELF_Rela +{ + public abstract Dictionary Entries { get; } +} + +class RelocationSectionAddend64 : RelocationSectionAddend +{ + public override Dictionary Entries { get; } = new Dictionary (); + + public RelocationSectionAddend64 (Section relaDynSection) + { + byte[] data = relaDynSection.GetContents (); + using var stream = new MemoryStream (data); + using var reader = new BinaryReader (stream); + + while (stream.Position < stream.Length) { + var entry = new ELF64_Rela (reader, (ulong)stream.Position); + Entries.Add (entry.r_offset, entry); + } + } +} diff --git a/tools/apput/src/Native/RelocationTypes.cs b/tools/apput/src/Native/RelocationTypes.cs new file mode 100644 index 00000000000..9323c076f46 --- /dev/null +++ b/tools/apput/src/Native/RelocationTypes.cs @@ -0,0 +1,118 @@ +namespace ApplicationUtility; + +enum RelocationTypeARM64 +{ + R_AARCH64_TLSGD_MOVW_G1 = 515, // GOT-rel. MOV{N,Z} 31:16. + R_AARCH64_TLSGD_MOVW_G0_NC = 516, // GOT-rel. MOVK imm. 15:0. + R_AARCH64_TLSLD_ADR_PREL21 = 517, // Like 512; local dynamic model. + R_AARCH64_TLSLD_ADR_PAGE21 = 518, // Like 513; local dynamic model. + R_AARCH64_TLSLD_ADD_LO12_NC = 519, // Like 514; local dynamic model. + R_AARCH64_TLSLD_MOVW_G1 = 520, // Like 515; local dynamic model. + R_AARCH64_TLSLD_MOVW_G0_NC = 521, // Like 516; local dynamic model. + R_AARCH64_TLSLD_LD_PREL19 = 522, // TLS PC-rel. load imm. 20:2. + R_AARCH64_TLSLD_MOVW_DTPREL_G2 = 523, // TLS DTP-rel. MOV{N,Z} 47:32. + R_AARCH64_TLSLD_MOVW_DTPREL_G1 = 524, // TLS DTP-rel. MOV{N,Z} 31:16. + R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC = 525, // Likewise; MOVK; no check. + R_AARCH64_TLSLD_MOVW_DTPREL_G0 = 526, // TLS DTP-rel. MOV{N,Z} 15:0. + R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC = 527, // Likewise; MOVK; no check. + R_AARCH64_TLSLD_ADD_DTPREL_HI12 = 528, // DTP-rel. ADD imm. from 23:12. + R_AARCH64_TLSLD_ADD_DTPREL_LO12 = 529, // DTP-rel. ADD imm. from 11:0. + R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC = 530, // Likewise; no ovfl. check. + R_AARCH64_TLSLD_LDST8_DTPREL_LO12 = 531, // DTP-rel. LD/ST imm. 11:0. + R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC = 532, // Likewise; no check. + R_AARCH64_TLSLD_LDST16_DTPREL_LO12 = 533, // DTP-rel. LD/ST imm. 11:1. + R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC = 534, // Likewise; no check. + R_AARCH64_TLSLD_LDST32_DTPREL_LO12 = 535, // DTP-rel. LD/ST imm. 11:2. + R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC = 536, // Likewise; no check. + R_AARCH64_TLSLD_LDST64_DTPREL_LO12 = 537, // DTP-rel. LD/ST imm. 11:3. + R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC = 538, // Likewise; no check. + R_AARCH64_TLSIE_MOVW_GOTTPREL_G1 = 539, // GOT-rel. MOV{N,Z} 31:16. + R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC = 540, // GOT-rel. MOVK 15:0. + R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 = 541, // Page-rel. ADRP 32:12. + R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC = 542, // Direct LD off. 11:3. + R_AARCH64_TLSIE_LD_GOTTPREL_PREL19 = 543, // PC-rel. load imm. 20:2. + R_AARCH64_TLSLE_MOVW_TPREL_G2 = 544, // TLS TP-rel. MOV{N,Z} 47:32. + R_AARCH64_TLSLE_MOVW_TPREL_G1 = 545, // TLS TP-rel. MOV{N,Z} 31:16. + R_AARCH64_TLSLE_MOVW_TPREL_G1_NC = 546, // Likewise; MOVK; no check. + R_AARCH64_TLSLE_MOVW_TPREL_G0 = 547, // TLS TP-rel. MOV{N,Z} 15:0. + R_AARCH64_TLSLE_MOVW_TPREL_G0_NC = 548, // Likewise; MOVK; no check. + R_AARCH64_TLSLE_ADD_TPREL_HI12 = 549, // TP-rel. ADD imm. 23:12. + R_AARCH64_TLSLE_ADD_TPREL_LO12 = 550, // TP-rel. ADD imm. 11:0. + R_AARCH64_TLSLE_ADD_TPREL_LO12_NC = 551, // Likewise; no ovfl. check. + R_AARCH64_TLSLE_LDST8_TPREL_LO12 = 552, // TP-rel. LD/ST off. 11:0. + R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC = 553, // Likewise; no ovfl. check. + R_AARCH64_TLSLE_LDST16_TPREL_LO12 = 554, // TP-rel. LD/ST off. 11:1. + R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC = 555, // Likewise; no check. + R_AARCH64_TLSLE_LDST32_TPREL_LO12 = 556, // TP-rel. LD/ST off. 11:2. + R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC = 557, // Likewise; no check. + R_AARCH64_TLSLE_LDST64_TPREL_LO12 = 558, // TP-rel. LD/ST off. 11:3. + R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC = 559, // Likewise; no check. + R_AARCH64_TLSDESC_LD_PREL19 = 560, // PC-rel. load immediate 20:2. + R_AARCH64_TLSDESC_ADR_PREL21 = 561, // PC-rel. ADR immediate 20:0. + R_AARCH64_TLSDESC_ADR_PAGE21 = 562, // Page-rel. ADRP imm. 32:12. + R_AARCH64_TLSDESC_LD64_LO12 = 563, // Direct LD off. from 11:3. + R_AARCH64_TLSDESC_ADD_LO12 = 564, // Direct ADD imm. from 11:0. + R_AARCH64_TLSDESC_OFF_G1 = 565, // GOT-rel. MOV{N,Z} imm. 31:16. + R_AARCH64_TLSDESC_OFF_G0_NC = 566, // GOT-rel. MOVK imm. 15:0; no ck. + R_AARCH64_TLSDESC_LDR = 567, // Relax LDR. + R_AARCH64_TLSDESC_ADD = 568, // Relax ADD. + R_AARCH64_TLSDESC_CALL = 569, // Relax BLR. + R_AARCH64_TLSLE_LDST128_TPREL_LO12 = 570, // TP-rel. LD/ST off. 11:4. + R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC = 571, // Likewise; no check. + R_AARCH64_TLSLD_LDST128_DTPREL_LO12 = 572, // DTP-rel. LD/ST imm. 11:4. + R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC = 573, // Likewise; no check. + R_AARCH64_COPY = 1024, // Copy symbol at runtime. + R_AARCH64_GLOB_DAT = 1025, // Create GOT entry. + R_AARCH64_JUMP_SLOT = 1026, // Create PLT entry. + R_AARCH64_RELATIVE = 1027, // Adjust by program base. + R_AARCH64_TLS_DTPMOD = 1028, // Module number, 64 bit. + R_AARCH64_TLS_DTPREL = 1029, // Module-relative offset, 64 bit. + R_AARCH64_TLS_TPREL = 1030, // TP-relative offset, 64 bit. + R_AARCH64_TLSDESC = 1031, // TLS Descriptor. + R_AARCH64_IRELATIVE = 1032, // STT_GNU_IFUNC relocation. +} + +enum RelocationTypeX64 +{ + R_X86_64_NONE = 0, // No reloc + R_X86_64_64 = 1, // Direct 64 bit + R_X86_64_PC32 = 2, // PC relative 32 bit signed + R_X86_64_GOT32 = 3, // 32 bit GOT entry + R_X86_64_PLT32 = 4, // 32 bit PLT address + R_X86_64_COPY = 5, // Copy symbol at runtime + R_X86_64_GLOB_DAT = 6, // Create GOT entry + R_X86_64_JUMP_SLOT = 7, // Create PLT entry + R_X86_64_RELATIVE = 8, // Adjust by program base + R_X86_64_GOTPCREL = 9, // 32 bit signed PC relative offset to GOT + R_X86_64_32 = 10, // Direct 32 bit zero extended + R_X86_64_32S = 11, // Direct 32 bit sign extended + R_X86_64_16 = 12, // Direct 16 bit zero extended + R_X86_64_PC16 = 13, // 16 bit sign extended pc relative + R_X86_64_8 = 14, // Direct 8 bit sign extended + R_X86_64_PC8 = 15, // 8 bit sign extended pc relative + R_X86_64_DTPMOD64 = 16, // ID of module containing symbol + R_X86_64_DTPOFF64 = 17, // Offset in module's TLS block + R_X86_64_TPOFF64 = 18, // Offset in initial TLS block + R_X86_64_TLSGD = 19, // 32 bit signed PC relative offset to two GOT entries for GD symbol + R_X86_64_TLSLD = 20, // 32 bit signed PC relative offset to two GOT entries for LD symbol + R_X86_64_DTPOFF32 = 21, // Offset in TLS block + R_X86_64_GOTTPOFF = 22, // 32 bit signed PC relative offset to GOT entry for IE symbol + R_X86_64_TPOFF32 = 23, // Offset in initial TLS block + R_X86_64_PC64 = 24, // PC relative 64 bit + R_X86_64_GOTOFF64 = 25, // 64 bit offset to GOT + R_X86_64_GOTPC32 = 26, // 32 bit signed pc relative offset to GOT + R_X86_64_GOT64 = 27, // 64-bit GOT entry offset + R_X86_64_GOTPCREL64 = 28, // 64-bit PC relative offset to GOT entry + R_X86_64_GOTPC64 = 29, // 64-bit PC relative offset to GOT + R_X86_64_GOTPLT64 = 30, // like GOT64, says PLT entry needed + R_X86_64_PLTOFF64 = 31, // 64-bit GOT relative offset to PLT entry + R_X86_64_SIZE32 = 32, // Size of symbol plus 32-bit addend + R_X86_64_SIZE64 = 33, // Size of symbol plus 64-bit addend + R_X86_64_GOTPC32_TLSDESC = 34, // GOT offset for TLS descriptor. + R_X86_64_TLSDESC_CALL = 35, // Marker for call through TLS descriptor. + R_X86_64_TLSDESC = 36, // TLS descriptor. + R_X86_64_IRELATIVE = 37, // Adjust indirectly by program base + R_X86_64_RELATIVE64 = 38, // 64-bit adjust by program base + R_X86_64_GOTPCRELX = 41, // Load from 32 bit signed pc relative offset to GOT entry without REX prefix, relaxable. + R_X86_64_REX_GOTPCRELX = 42, // Load from 32 bit signed pc relative offset to GOT entry with REX prefix, relaxable. +} diff --git a/tools/apput/src/Native/SharedLibrary.cs b/tools/apput/src/Native/SharedLibrary.cs index 64ba7490ad2..b85be5b4955 100644 --- a/tools/apput/src/Native/SharedLibrary.cs +++ b/tools/apput/src/Native/SharedLibrary.cs @@ -7,7 +7,7 @@ using ApplicationUtility; -public class SharedLibrary : IAspect, IDisposable +class SharedLibrary : IAspect, IDisposable { const uint ELF_MAGIC = 0x464c457f; @@ -24,7 +24,7 @@ public class SharedLibrary : IAspect, IDisposable IELF elf; bool disposed; - SharedLibrary (Stream stream, string libraryName) + protected SharedLibrary (Stream stream, string libraryName) { this.libraryStream = stream; this.libraryName = libraryName; @@ -80,7 +80,7 @@ public Stream OpenAndroidPayload () return new SubStream (libraryStream, (long)payloadOffset, (long)payloadSize); } - static bool IsSupportedELFSharedLibrary (Stream stream, string? description) + protected static bool IsSupportedELFSharedLibrary (Stream stream, string? description) { if (stream.Length < 4) { // Less than that and we know there isn't room for ELF magic Log.Debug ($"SharedLibrary: stream ('{description}') is too short to be an ELF image."); diff --git a/tools/apput/src/Package/ApplicationPackage.cs b/tools/apput/src/Package/ApplicationPackage.cs index f3de6bdc38f..20d8edfb2b3 100644 --- a/tools/apput/src/Package/ApplicationPackage.cs +++ b/tools/apput/src/Package/ApplicationPackage.cs @@ -49,6 +49,7 @@ public abstract class ApplicationPackage : IAspect public string MainActivity { get; protected set; } = ""; public List? AssemblyStores { get; protected set; } public List Architectures { get; protected set; } = new (); + public List NativeAppInfos { get; protected set; } = new (); AndroidManifest? manifest; @@ -85,6 +86,7 @@ public static IAspect LoadAspect (Stream stream, IAspectState state, string? des ret.TryDetectWhetherIsSigned (); ret.TryLoadAssemblyStores (); ret.TryLoadAndroidManifest (); + ret.TryLoadXamarinAppLibraries (); return ret; } @@ -140,6 +142,41 @@ void TryDetectRuntime () // some public symbols to verify that. } + void TryLoadXamarinAppLibraries () + { + foreach (AndroidTargetArch arch in Architectures) { + string libPath = GetNativeLibFile (arch, "libxamarin-app.so"); + LibXamarinApp? lib = TryLoadLibXamarinApp (libPath); + if (lib == null) { + continue; + } + NativeAppInfos.Add (new NativeAppInfo (lib)); + } + } + + LibXamarinApp? TryLoadLibXamarinApp (string libPath) + { + Stream? libStream = TryGetEntryStream (libPath); + if (libStream == null) { + return null; + } + + string fullLibPath = $"{Description}@!{libPath}"; + try { + IAspectState state = LibXamarinApp.ProbeAspect (libStream, fullLibPath); + if (!state.Success) { + Log.Debug ($"Assembly store '{libPath}' is not in a supported format"); + libStream.Close (); + return null; + } + + return (LibXamarinApp)LibXamarinApp.LoadAspect (libStream, state, fullLibPath); + } catch (Exception ex) { + Log.Debug ($"Failed to load Xamarin app library '{libPath}'. Exception thrown:", ex); + return null; + } + } + void TryDetectWhetherIsSigned () { Signed = HasAnyEntries (Zip, KnownSignatureEntries);