From 2b12482627ec3c1ff73c9347e8c0a5180794de54 Mon Sep 17 00:00:00 2001 From: Ada Date: Mon, 5 Feb 2024 03:37:03 +0000 Subject: [PATCH] Refactor Oodle and Dll Prep for better Linux and macOS support --- CUE4Parse-Conversion/Constants.cs | 3 +- CUE4Parse-Conversion/Textures/BC/Detex.cs | 37 ++-- .../Textures/PlatformDeswizzlers.cs | 44 ++-- CUE4Parse/Compression/Oodle.cs | 190 +++++++++++++++--- 4 files changed, 197 insertions(+), 77 deletions(-) diff --git a/CUE4Parse-Conversion/Constants.cs b/CUE4Parse-Conversion/Constants.cs index b99b2777a..8c36fc017 100644 --- a/CUE4Parse-Conversion/Constants.cs +++ b/CUE4Parse-Conversion/Constants.cs @@ -22,5 +22,6 @@ public class Constants public const int DXT_BITS_PER_PIXEL = 4; - public const string DETEX_DLL_NAME = "Detex.dll"; + public const string DETEX_DLL_NAME = "Detex"; + public const string TEGRA_SWIZZLE_DLL_NAME = "tegra_swizzle_x64"; } diff --git a/CUE4Parse-Conversion/Textures/BC/Detex.cs b/CUE4Parse-Conversion/Textures/BC/Detex.cs index 4e499cb1f..6386cc972 100644 --- a/CUE4Parse-Conversion/Textures/BC/Detex.cs +++ b/CUE4Parse-Conversion/Textures/BC/Detex.cs @@ -57,39 +57,32 @@ public static bool DecodeDetexLinear(byte[] inp, byte[] dst, int width, int heig [DllImport(Constants.DETEX_DLL_NAME)] private static extern unsafe bool detexDecompressTextureLinear(detexTexture* texture, byte* pixelBuffer, uint pixelFormat); - + private static void PrepareDllFile() { + if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("CUE4Parse_Conversion.Resources.Detex.dll"); if (stream == null) throw new MissingManifestResourceException("Couldn't find Detex.dll in Embedded Resources"); + var ba = new byte[(int)stream.Length]; - stream.Read(ba, 0, (int)stream.Length); - - bool fileOk; - var dllFile = Constants.DETEX_DLL_NAME; + stream.ReadExactly(ba); - using (var sha1 = new SHA1CryptoServiceProvider()) + var dllFile = Constants.DETEX_DLL_NAME + ".dll"; + + if (File.Exists(dllFile)) { - var fileHash = BitConverter.ToString(sha1.ComputeHash(ba)).Replace("-", string.Empty); + var bb = File.ReadAllBytes(dllFile); + var fileHash = SHA1.HashData(ba).AsSpan(); + var fileHash2 = SHA1.HashData(bb).AsSpan(); - if (File.Exists(dllFile)) - { - var bb = File.ReadAllBytes(dllFile); - var fileHash2 = BitConverter.ToString(sha1.ComputeHash(bb)).Replace("-", string.Empty); - - fileOk = fileHash == fileHash2; - } - else - { - fileOk = false; - } + if (fileHash.SequenceEqual(fileHash2)) + return; } - if (!fileOk) - { - File.WriteAllBytes(dllFile, ba); - } + File.WriteAllBytes(dllFile, ba); } } } diff --git a/CUE4Parse-Conversion/Textures/PlatformDeswizzlers.cs b/CUE4Parse-Conversion/Textures/PlatformDeswizzlers.cs index ff490160e..a5f16f68a 100644 --- a/CUE4Parse-Conversion/Textures/PlatformDeswizzlers.cs +++ b/CUE4Parse-Conversion/Textures/PlatformDeswizzlers.cs @@ -15,49 +15,43 @@ static PlatformDeswizzlers() PrepareDllFile(); } - [DllImport("tegra_swizzle_x64", EntryPoint = "deswizzle_block_linear")] + [DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "deswizzle_block_linear")] private static extern unsafe void DeswizzleBlockLinearX64(ulong width, ulong height, ulong depth, byte* source, ulong sourceLength, byte[] destination, ulong destinationLength, ulong blockHeight, ulong bytesPerPixel); - [DllImport("tegra_swizzle_x64", EntryPoint = "swizzled_surface_size")] + [DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "swizzled_surface_size")] private static extern ulong GetSurfaceSizeX64(ulong width, ulong height, ulong depth, ulong blockHeight, ulong bytesPerPixel); - [DllImport("tegra_swizzle_x64", EntryPoint = "block_height_mip0")] + [DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "block_height_mip0")] private static extern ulong BlockHeightMip0X64(ulong height); - [DllImport("tegra_swizzle_x64", EntryPoint = "mip_block_height")] + [DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "mip_block_height")] private static extern ulong MipBlockHeightX64(ulong mipHeight, ulong blockHeightMip0); private static void PrepareDllFile() { + if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + return; + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("CUE4Parse_Conversion.Resources.tegra_swizzle_x64.dll"); if (stream == null) throw new MissingManifestResourceException("Couldn't find tegra_swizzle_x64.dll in Embedded Resources"); - var ba = new byte[(int) stream.Length]; - stream.Read(ba, 0, (int) stream.Length); + + var ba = new byte[(int)stream.Length]; + stream.ReadExactly(ba); - bool fileOk; - - using (var sha1 = SHA1.Create()) + var dllFile = Constants.TEGRA_SWIZZLE_DLL_NAME + ".dll"; + + if (File.Exists(dllFile)) { - var fileHash = BitConverter.ToString(sha1.ComputeHash(ba)).Replace("-", string.Empty); - - if (File.Exists("tegra_swizzle_x64.dll")) - { - var bb = File.ReadAllBytes("tegra_swizzle_x64.dll"); - var fileHash2 = BitConverter.ToString(sha1.ComputeHash(bb)).Replace("-", string.Empty); + var bb = File.ReadAllBytes(dllFile); + var fileHash = SHA1.HashData(ba).AsSpan(); + var fileHash2 = SHA1.HashData(bb).AsSpan(); - fileOk = fileHash == fileHash2; - } - else - { - fileOk = false; - } + if (fileHash.SequenceEqual(fileHash2)) + return; } - if (!fileOk) - { - File.WriteAllBytes("tegra_swizzle_x64.dll", ba); - } + File.WriteAllBytes(dllFile, ba); } public static byte[] GetDeswizzledData(byte[] data, FTexture2DMipMap mip, FPixelFormatInfo formatInfo) diff --git a/CUE4Parse/Compression/Oodle.cs b/CUE4Parse/Compression/Oodle.cs index 7b521cd6b..190dee8d4 100644 --- a/CUE4Parse/Compression/Oodle.cs +++ b/CUE4Parse/Compression/Oodle.cs @@ -5,6 +5,8 @@ using Serilog; using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Http; using System.Runtime.InteropServices; @@ -14,68 +16,194 @@ namespace CUE4Parse.Compression { public class OodleException : ParserException { - public OodleException(string? message = null, Exception? innerException = null) : base(message, innerException) { } public OodleException(FArchive reader, string? message = null, Exception? innerException = null) : base(reader, message, innerException) { } + + public OodleException(string? message, Exception? innerException) : base(message, innerException) { } + + public OodleException(string message) : base(message) { } + + public OodleException() : base("Oodle decompression failed") { } } public static class Oodle { - public unsafe delegate long OodleDecompress(byte* bufferPtr, long bufferSize, byte* outputPtr, long outputSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule); - private const string WARFRAME_CONTENT_HOST = "https://content.warframe.com"; private const string WARFRAME_ORIGIN_HOST = "https://origin.warframe.com"; private const string WARFRAME_INDEX_PATH = "/origin/50F7040A/index.txt.lzma"; private const string WARFRAME_INDEX_URL = WARFRAME_ORIGIN_HOST + WARFRAME_INDEX_PATH; - public const string OODLE_DLL_NAME = "oo2core_9_win64.dll"; - public static OodleDecompress DecompressFunc; + // this will return a platform-appropriate library name, wildcarded to suppress prefixes, suffixes and version masks + // - oo2core_9_win32.dll + // - oo2core_9_win64.dll + // - oo2core_9_winuwparm64.dll + // - liboo2coremac64.2.9.10.dylib + // - liboo2corelinux64.so.9 + // - liboo2corelinuxarm64.so.9 + // - liboo2corelinuxarm32.so.9 + public static IEnumerable OodleLibName + { + get + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) + yield return "*oo2core*winuwparm64*.so*"; + else if (RuntimeInformation.ProcessArchitecture == Architecture.X86) + yield return "*oo2core*win32*.so*"; + + yield return "*oo2core*win64*.so*"; + + yield break; + } + + // you can find these in the unreal source post-installation + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) + yield return "*oo2core*linuxarm64*.so*"; + else if (RuntimeInformation.ProcessArchitecture == Architecture.Arm || + RuntimeInformation.ProcessArchitecture == Architecture.Armv6) + yield return "*oo2core*linuxarm32*.so*"; + + yield return "*oo2core*linux64*.so*"; + + yield break; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) + yield return "*oo2core*macarm64*.dylib"; // todo: this doesn't exist. + + yield return "*oo2core*mac64*.dylib"; + + yield break; + } + + throw new PlatformNotSupportedException(); + } + } - static unsafe Oodle() + public static bool IsReady => DecompressDelegate != null; + public static OodleLZ_Decompress? DecompressDelegate { get; set; } + + public static bool TryFindOodleDll(string? path, [MaybeNullWhen(false)] out string result) { - DecompressFunc = OodleLZ_Decompress; + path ??= Environment.CurrentDirectory; + foreach (var oodleLibName in OodleLibName) + { + var files = Directory.GetFiles(path, oodleLibName, SearchOption.TopDirectoryOnly); + if (files.Length == 0) + continue; + + result = files[0]; + return true; + } + + result = null; + return false; } public static bool LoadOodleDll(string? path = null) { - if (File.Exists(OODLE_DLL_NAME)) return true; - return DownloadOodleDll(path).GetAwaiter().GetResult(); + if (IsReady) + return true; + + path ??= Environment.CurrentDirectory; + + if (Directory.Exists(path) && new FileInfo(path).Attributes.HasFlag(FileAttributes.Directory)) + { + if (!TryFindOodleDll(path, out var oodlePath)) + { + if (!DownloadOodleDll().GetAwaiter().GetResult() || !TryFindOodleDll(path, out oodlePath)) + return false; + } + + path = oodlePath; + } + + if (string.IsNullOrEmpty(path) || !File.Exists(path)) + return false; + + if (!NativeLibrary.TryLoad(path, out var handle)) + return false; + + if (!NativeLibrary.TryGetExport(handle, nameof(OodleLZ_Decompress), out var address)) + return false; + + DecompressDelegate = Marshal.GetDelegateForFunctionPointer(address); + return true; } - public static unsafe void Decompress(byte[] compressed, int compressedOffset, int compressedSize, - byte[] uncompressed, int uncompressedOffset, int uncompressedSize, FArchive? reader = null) + public static unsafe void Decompress(Memory input, int inputOffset, int inputSize, + Memory output, int outputOffset, int outputSize, FArchive? reader = null) { - if (DecompressFunc == OodleLZ_Decompress) + if (!IsReady) LoadOodleDll(); - - long decodedSize; - - fixed (byte* compressedPtr = compressed, uncompressedPtr = uncompressed) + + if (DecompressDelegate == null) { - decodedSize = DecompressFunc(compressedPtr + compressedOffset, compressedSize, - uncompressedPtr + uncompressedOffset, uncompressedSize, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3); + if (reader != null) + throw new OodleException(reader, "Oodle library not loaded"); + + throw new OodleException("Oodle library not loaded"); } + var inputSlice = input.Slice(inputOffset, inputSize); + var outputSlice = output.Slice(outputOffset, outputSize); + using var inPin = inputSlice.Pin(); + using var outPin = outputSlice.Pin(); + + var decodedSize = DecompressDelegate(inPin.Pointer, inputSlice.Length, outPin.Pointer, outputSlice.Length); + if (decodedSize <= 0) { - if (reader != null) throw new OodleException(reader, $"Oodle decompression failed with result {decodedSize}"); + if (reader != null) + throw new OodleException(reader, $"Oodle decompression failed with result {decodedSize}"); + throw new OodleException($"Oodle decompression failed with result {decodedSize}"); } - if (decodedSize < uncompressedSize) + if (decodedSize < outputSize) { // Not sure whether this should be an exception or not - Log.Warning("Oodle decompression just decompressed {0} bytes of the expected {1} bytes", decodedSize, uncompressedSize); + Log.Warning("Oodle decompression just decompressed {0} bytes of the expected {1} bytes", decodedSize, outputSize); } } - [DllImport(OODLE_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] - public static extern long OodleLZ_Decompress(byte[] buffer, long bufferSize, byte[] output, long outputBufferSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule); - [DllImport(OODLE_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] - public static extern unsafe long OodleLZ_Decompress(byte* buffer, long bufferSize, byte* output, long outputBufferSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule); + public enum OodleLZ_Decode_ThreadPhase + { + ThreadPhase1 = 1, + ThreadPhase2 = 2, + ThreadPhaseAll = 3, + Unthreaded = ThreadPhaseAll, + } + + public enum OodleLZ_Verbosity + { + None = 0, + Minimal = 1, + Some = 2, + Lots = 3, + } + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public unsafe delegate int OodleLZ_Decompress(void* srcBuf, int srcSize, void* rawBuf, int rawSize, + int fuzzSafe = 1, int checkCRC = 0, OodleLZ_Verbosity verbosity = OodleLZ_Verbosity.None, + void* decBufBase = null, int decBufSize = 0, void* fpCallback = null, void* callbackUserData = null, + void* decoderMemory = null, int decoderMemorySize = 0, + OodleLZ_Decode_ThreadPhase threadPhase = OodleLZ_Decode_ThreadPhase.Unthreaded); - public static async Task DownloadOodleDll(string? path) + public static async Task DownloadOodleDll(string? path = null) { - using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(5) }; + if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64) + { + Log.Warning("Cannot download Oodle library for non-64-bit windows or non-windows"); + return false; + } + + using var client = new HttpClient(); + client.Timeout = TimeSpan.FromSeconds(5); try { using var indexResponse = await client.GetAsync(WARFRAME_INDEX_URL).ConfigureAwait(false); @@ -92,7 +220,7 @@ public static async Task DownloadOodleDll(string? path) var line = await indexReader.ReadLineAsync().ConfigureAwait(false); if (string.IsNullOrEmpty(line)) continue; - if (line.Contains(OODLE_DLL_NAME)) + if (line.Contains("Oodle/x64/final/oo2core_")) { dllUrl = WARFRAME_CONTENT_HOST + line[..line.IndexOf(',')]; break; @@ -105,13 +233,16 @@ public static async Task DownloadOodleDll(string? path) return false; } + var dllName = dllUrl[(dllUrl.LastIndexOf('/') + 1)..]; + dllName = dllName[..dllName.IndexOf('.')] + ".dll"; + using var dllResponse = await client.GetAsync(dllUrl).ConfigureAwait(false); await using var dllLzmaStream = await dllResponse.Content.ReadAsStreamAsync().ConfigureAwait(false); await using var dllStream = new MemoryStream(); Lzma.Decompress(dllLzmaStream, dllStream); dllStream.Position = 0; - var dllPath = path ?? OODLE_DLL_NAME; + var dllPath = path ?? dllName; var dllFs = File.Create(dllPath); await dllStream.CopyToAsync(dllFs).ConfigureAwait(false); await dllFs.DisposeAsync().ConfigureAwait(false); @@ -122,6 +253,7 @@ public static async Task DownloadOodleDll(string? path) { Log.Warning(e, "Uncaught exception while downloading oodle dll"); } + return false; } }