Skip to content

Commit

Permalink
Refactor Oodle and Dll Prep for better Linux and macOS support
Browse files Browse the repository at this point in the history
  • Loading branch information
@yretenai
yretenai committed Feb 5, 2024
1 parent 20662a9 commit c990a14
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 77 deletions.
3 changes: 2 additions & 1 deletion CUE4Parse-Conversion/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ public class Constants

public const int DXT_BITS_PER_PIXEL = 4;

public const string DETEX_DLL_NAME = "Detex.dll";
public const string DETEX_DLL_NAME = "Detex";
public const string TEGRA_SWIZZLE_DLL_NAME = "tegra_swizzle_x64";
}
37 changes: 15 additions & 22 deletions CUE4Parse-Conversion/Textures/BC/Detex.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,39 +57,32 @@ public static bool DecodeDetexLinear(byte[] inp, byte[] dst, int width, int heig
[DllImport(Constants.DETEX_DLL_NAME)]
private static extern unsafe bool detexDecompressTextureLinear(detexTexture* texture, byte* pixelBuffer,
uint pixelFormat);

private static void PrepareDllFile()
{
if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64)
return;

using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("CUE4Parse_Conversion.Resources.Detex.dll");
if (stream == null)
throw new MissingManifestResourceException("Couldn't find Detex.dll in Embedded Resources");

var ba = new byte[(int)stream.Length];
stream.Read(ba, 0, (int)stream.Length);

bool fileOk;
var dllFile = Constants.DETEX_DLL_NAME;
stream.ReadExactly(ba);

using (var sha1 = new SHA1CryptoServiceProvider())
var dllFile = Constants.DETEX_DLL_NAME + ".dll";

if (File.Exists(dllFile))
{
var fileHash = BitConverter.ToString(sha1.ComputeHash(ba)).Replace("-", string.Empty);
var bb = File.ReadAllBytes(dllFile);
var fileHash = SHA1.HashData(ba).AsSpan();
var fileHash2 = SHA1.HashData(bb).AsSpan();

if (File.Exists(dllFile))
{
var bb = File.ReadAllBytes(dllFile);
var fileHash2 = BitConverter.ToString(sha1.ComputeHash(bb)).Replace("-", string.Empty);

fileOk = fileHash == fileHash2;
}
else
{
fileOk = false;
}
if (fileHash.SequenceEqual(fileHash2))
return;
}

if (!fileOk)
{
File.WriteAllBytes(dllFile, ba);
}
File.WriteAllBytes(dllFile, ba);
}
}
}
44 changes: 19 additions & 25 deletions CUE4Parse-Conversion/Textures/PlatformDeswizzlers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,49 +15,43 @@ static PlatformDeswizzlers()
PrepareDllFile();
}

[DllImport("tegra_swizzle_x64", EntryPoint = "deswizzle_block_linear")]
[DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "deswizzle_block_linear")]
private static extern unsafe void DeswizzleBlockLinearX64(ulong width, ulong height, ulong depth, byte* source, ulong sourceLength, byte[] destination, ulong destinationLength, ulong blockHeight, ulong bytesPerPixel);

[DllImport("tegra_swizzle_x64", EntryPoint = "swizzled_surface_size")]
[DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "swizzled_surface_size")]
private static extern ulong GetSurfaceSizeX64(ulong width, ulong height, ulong depth, ulong blockHeight, ulong bytesPerPixel);

[DllImport("tegra_swizzle_x64", EntryPoint = "block_height_mip0")]
[DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "block_height_mip0")]
private static extern ulong BlockHeightMip0X64(ulong height);

[DllImport("tegra_swizzle_x64", EntryPoint = "mip_block_height")]
[DllImport(Constants.TEGRA_SWIZZLE_DLL_NAME, EntryPoint = "mip_block_height")]
private static extern ulong MipBlockHeightX64(ulong mipHeight, ulong blockHeightMip0);

private static void PrepareDllFile()
{
if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64)
return;

using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("CUE4Parse_Conversion.Resources.tegra_swizzle_x64.dll");
if (stream == null)
throw new MissingManifestResourceException("Couldn't find tegra_swizzle_x64.dll in Embedded Resources");
var ba = new byte[(int) stream.Length];
stream.Read(ba, 0, (int) stream.Length);

var ba = new byte[(int)stream.Length];
stream.ReadExactly(ba);

bool fileOk;

using (var sha1 = SHA1.Create())
var dllFile = Constants.TEGRA_SWIZZLE_DLL_NAME + ".dll";
if (File.Exists(dllFile))
{
var fileHash = BitConverter.ToString(sha1.ComputeHash(ba)).Replace("-", string.Empty);

if (File.Exists("tegra_swizzle_x64.dll"))
{
var bb = File.ReadAllBytes("tegra_swizzle_x64.dll");
var fileHash2 = BitConverter.ToString(sha1.ComputeHash(bb)).Replace("-", string.Empty);
var bb = File.ReadAllBytes(dllFile);
var fileHash = SHA1.HashData(ba).AsSpan();
var fileHash2 = SHA1.HashData(bb).AsSpan();

fileOk = fileHash == fileHash2;
}
else
{
fileOk = false;
}
if (fileHash.SequenceEqual(fileHash2))
return;
}

if (!fileOk)
{
File.WriteAllBytes("tegra_swizzle_x64.dll", ba);
}
File.WriteAllBytes(dllFile, ba);
}

public static byte[] GetDeswizzledData(byte[] data, FTexture2DMipMap mip, FPixelFormatInfo formatInfo)
Expand Down
190 changes: 161 additions & 29 deletions CUE4Parse/Compression/Oodle.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
using Serilog;

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Net.Http;
using System.Runtime.InteropServices;
Expand All @@ -14,68 +16,194 @@ namespace CUE4Parse.Compression
{
public class OodleException : ParserException
{
public OodleException(string? message = null, Exception? innerException = null) : base(message, innerException) { }
public OodleException(FArchive reader, string? message = null, Exception? innerException = null) : base(reader, message, innerException) { }

public OodleException(string? message, Exception? innerException) : base(message, innerException) { }

public OodleException(string message) : base(message) { }

public OodleException() : base("Oodle decompression failed") { }
}

public static class Oodle
{
public unsafe delegate long OodleDecompress(byte* bufferPtr, long bufferSize, byte* outputPtr, long outputSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule);

private const string WARFRAME_CONTENT_HOST = "https://content.warframe.com";
private const string WARFRAME_ORIGIN_HOST = "https://origin.warframe.com";
private const string WARFRAME_INDEX_PATH = "/origin/50F7040A/index.txt.lzma";
private const string WARFRAME_INDEX_URL = WARFRAME_ORIGIN_HOST + WARFRAME_INDEX_PATH;
public const string OODLE_DLL_NAME = "oo2core_9_win64.dll";

public static OodleDecompress DecompressFunc;
// this will return a platform-appropriate library name, wildcarded to suppress prefixes, suffixes and version masks
// - oo2core_9_win32.dll
// - oo2core_9_win64.dll
// - oo2core_9_winuwparm64.dll
// - liboo2coremac64.2.9.10.dylib
// - liboo2corelinux64.so.9
// - liboo2corelinuxarm64.so.9
// - liboo2corelinuxarm32.so.9
public static IEnumerable<string> OodleLibName
{
get
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
yield return "*oo2core*winuwparm64*.dll";
else if (RuntimeInformation.ProcessArchitecture == Architecture.X86)
yield return "*oo2core*win32*.dll";

yield return "*oo2core*win64*.dll";

yield break;
}

// you can find these in the unreal source post-installation
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
yield return "*oo2core*linuxarm64*.so*";
else if (RuntimeInformation.ProcessArchitecture == Architecture.Arm ||
RuntimeInformation.ProcessArchitecture == Architecture.Armv6)
yield return "*oo2core*linuxarm32*.so*";

yield return "*oo2core*linux64*.so*";

yield break;
}

if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
yield return "*oo2core*macarm64*.dylib"; // todo: this doesn't exist.

yield return "*oo2core*mac64*.dylib";

yield break;
}

throw new PlatformNotSupportedException();
}
}

static unsafe Oodle()
public static bool IsReady => DecompressDelegate != null;
public static OodleLZ_Decompress? DecompressDelegate { get; set; }

public static bool TryFindOodleDll(string? path, [MaybeNullWhen(false)] out string result)
{
DecompressFunc = OodleLZ_Decompress;
path ??= Environment.CurrentDirectory;
foreach (var oodleLibName in OodleLibName)
{
var files = Directory.GetFiles(path, oodleLibName, SearchOption.TopDirectoryOnly);
if (files.Length == 0)
continue;

result = files[0];
return true;
}

result = null;
return false;
}

public static bool LoadOodleDll(string? path = null)
{
if (File.Exists(OODLE_DLL_NAME)) return true;
return DownloadOodleDll(path).GetAwaiter().GetResult();
if (IsReady)
return true;

path ??= Environment.CurrentDirectory;

if (Directory.Exists(path) && new FileInfo(path).Attributes.HasFlag(FileAttributes.Directory))
{
if (!TryFindOodleDll(path, out var oodlePath))
{
if (!DownloadOodleDll().GetAwaiter().GetResult() || !TryFindOodleDll(path, out oodlePath))
return false;
}

path = oodlePath;
}

if (string.IsNullOrEmpty(path) || !File.Exists(path))
return false;

if (!NativeLibrary.TryLoad(path, out var handle))
return false;

if (!NativeLibrary.TryGetExport(handle, nameof(OodleLZ_Decompress), out var address))
return false;

DecompressDelegate = Marshal.GetDelegateForFunctionPointer<OodleLZ_Decompress>(address);
return true;
}

public static unsafe void Decompress(byte[] compressed, int compressedOffset, int compressedSize,
byte[] uncompressed, int uncompressedOffset, int uncompressedSize, FArchive? reader = null)
public static unsafe void Decompress(Memory<byte> input, int inputOffset, int inputSize,
Memory<byte> output, int outputOffset, int outputSize, FArchive? reader = null)
{
if (DecompressFunc == OodleLZ_Decompress)
if (!IsReady)
LoadOodleDll();

long decodedSize;

fixed (byte* compressedPtr = compressed, uncompressedPtr = uncompressed)

if (DecompressDelegate == null)
{
decodedSize = DecompressFunc(compressedPtr + compressedOffset, compressedSize,
uncompressedPtr + uncompressedOffset, uncompressedSize, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3);
if (reader != null)
throw new OodleException(reader, "Oodle library not loaded");

throw new OodleException("Oodle library not loaded");
}

var inputSlice = input.Slice(inputOffset, inputSize);
var outputSlice = output.Slice(outputOffset, outputSize);
using var inPin = inputSlice.Pin();
using var outPin = outputSlice.Pin();

var decodedSize = DecompressDelegate(inPin.Pointer, inputSlice.Length, outPin.Pointer, outputSlice.Length);

if (decodedSize <= 0)
{
if (reader != null) throw new OodleException(reader, $"Oodle decompression failed with result {decodedSize}");
if (reader != null)
throw new OodleException(reader, $"Oodle decompression failed with result {decodedSize}");

throw new OodleException($"Oodle decompression failed with result {decodedSize}");
}

if (decodedSize < uncompressedSize)
if (decodedSize < outputSize)
{
// Not sure whether this should be an exception or not
Log.Warning("Oodle decompression just decompressed {0} bytes of the expected {1} bytes", decodedSize, uncompressedSize);
Log.Warning("Oodle decompression just decompressed {0} bytes of the expected {1} bytes", decodedSize, outputSize);
}
}

[DllImport(OODLE_DLL_NAME, CallingConvention = CallingConvention.Cdecl)]
public static extern long OodleLZ_Decompress(byte[] buffer, long bufferSize, byte[] output, long outputBufferSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule);
[DllImport(OODLE_DLL_NAME, CallingConvention = CallingConvention.Cdecl)]
public static extern unsafe long OodleLZ_Decompress(byte* buffer, long bufferSize, byte* output, long outputBufferSize, int a, int b, int c, long d, long e, long f, long g, long h, long i, int threadModule);
public enum OodleLZ_Decode_ThreadPhase
{
ThreadPhase1 = 1,
ThreadPhase2 = 2,
ThreadPhaseAll = 3,
Unthreaded = ThreadPhaseAll,
}

public enum OodleLZ_Verbosity
{
None = 0,
Minimal = 1,
Some = 2,
Lots = 3,
}

[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public unsafe delegate int OodleLZ_Decompress(void* srcBuf, int srcSize, void* rawBuf, int rawSize,
int fuzzSafe = 1, int checkCRC = 0, OodleLZ_Verbosity verbosity = OodleLZ_Verbosity.None,
void* decBufBase = null, int decBufSize = 0, void* fpCallback = null, void* callbackUserData = null,
void* decoderMemory = null, int decoderMemorySize = 0,
OodleLZ_Decode_ThreadPhase threadPhase = OodleLZ_Decode_ThreadPhase.Unthreaded);

public static async Task<bool> DownloadOodleDll(string? path)
public static async Task<bool> DownloadOodleDll(string? path = null)
{
using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(5) };
if (!OperatingSystem.IsWindows() || RuntimeInformation.ProcessArchitecture != Architecture.X64)
{
Log.Warning("Cannot download Oodle library for non-64-bit windows or non-windows");
return false;
}

using var client = new HttpClient();
client.Timeout = TimeSpan.FromSeconds(5);
try
{
using var indexResponse = await client.GetAsync(WARFRAME_INDEX_URL).ConfigureAwait(false);
Expand All @@ -92,7 +220,7 @@ public static async Task<bool> DownloadOodleDll(string? path)
var line = await indexReader.ReadLineAsync().ConfigureAwait(false);
if (string.IsNullOrEmpty(line)) continue;

if (line.Contains(OODLE_DLL_NAME))
if (line.Contains("Oodle/x64/final/oo2core_"))
{
dllUrl = WARFRAME_CONTENT_HOST + line[..line.IndexOf(',')];
break;
Expand All @@ -105,13 +233,16 @@ public static async Task<bool> DownloadOodleDll(string? path)
return false;
}

var dllName = dllUrl[(dllUrl.LastIndexOf('/') + 1)..];
dllName = dllName[..dllName.IndexOf('.')] + ".dll";

using var dllResponse = await client.GetAsync(dllUrl).ConfigureAwait(false);
await using var dllLzmaStream = await dllResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
await using var dllStream = new MemoryStream();

Lzma.Decompress(dllLzmaStream, dllStream);
dllStream.Position = 0;
var dllPath = path ?? OODLE_DLL_NAME;
var dllPath = path ?? dllName;
var dllFs = File.Create(dllPath);
await dllStream.CopyToAsync(dllFs).ConfigureAwait(false);
await dllFs.DisposeAsync().ConfigureAwait(false);
Expand All @@ -122,6 +253,7 @@ public static async Task<bool> DownloadOodleDll(string? path)
{
Log.Warning(e, "Uncaught exception while downloading oodle dll");
}

return false;
}
}
Expand Down

0 comments on commit c990a14

Please sign in to comment.