From 47595e64f48afc1f63e037f9506ef93bba41e81f Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 11 Dec 2024 10:12:56 -0700 Subject: [PATCH 01/22] Basic structure of the projects, can login via a test, and stay logged in --- .../DTOs/Depot.cs | 29 +++ .../DTOs/ProductInfo.cs | 19 ++ .../IAuthInterventionHandler.cs | 12 ++ .../IAuthStorage.cs | 17 ++ .../ISteamSession.cs | 15 ++ .../NexusMods.Abstractions.Steam.csproj | 7 + .../Values/AppId.cs | 12 ++ .../Values/DepotId.cs | 12 ++ .../Values/ManifestId.cs | 12 ++ Directory.Packages.props | 2 + NexusMods.App.sln | 31 +++ NexusMods.App.sln.DotSettings | 1 + NexusMods.Stores.Steam/CDNPool.cs | 9 + .../NexusMods.Stores.Steam.csproj | 20 ++ NexusMods.Stores.Steam/Program.cs | 12 ++ NexusMods.Stores.Steam/README.md | 6 + NexusMods.Stores.Steam/Session.cs | 202 ++++++++++++++++++ .../AppDirectoryAuthStorage.cs | 35 +++ .../DTOs/AuthData.cs | 20 ++ .../LoggingAuthInterventionHandler.cs | 22 ++ .../NexusMods.Networking.Steam.csproj | 20 ++ .../ProductInfoParser.cs | 16 ++ .../NexusMods.Networking.Steam/Services.cs | 35 +++ .../NexusMods.Networking.Steam/Session.cs | 176 +++++++++++++++ .../BasicApiTests.cs | 19 ++ .../NexusMods.Networking.Steam.Tests.csproj | 17 ++ .../Startup.cs | 18 ++ 27 files changed, 796 insertions(+) create mode 100644 Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/IAuthInterventionHandler.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/IAuthStorage.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj create mode 100644 Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs create mode 100644 NexusMods.Stores.Steam/CDNPool.cs create mode 100644 NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj create mode 100644 NexusMods.Stores.Steam/Program.cs create mode 100644 NexusMods.Stores.Steam/README.md create mode 100644 NexusMods.Stores.Steam/Session.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/DTOs/AuthData.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/LoggingAuthInterventionHandler.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj create mode 100644 src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/Services.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/Session.cs create mode 100644 tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs create mode 100644 tests/Networking/NexusMods.Networking.Steam.Tests/NexusMods.Networking.Steam.Tests.csproj create mode 100644 tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs new file mode 100644 index 0000000000..26c2e1fe71 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs @@ -0,0 +1,29 @@ +using NexusMods.Abstractions.Steam.Values; + +namespace NexusMods.Abstractions.Steam.DTOs; + +/// +/// Information about a depot on Steam. +/// +public class Depot +{ + /// + /// The name of the depot. + /// + public required string Name { get; init; } + + /// + /// The app id associated with the depot. + /// + public required AppId AppId { get; init; } + + /// + /// The id of the depot. + /// + public required DepotId DepotId { get; init; } + + /// + /// The Current ManifestId of the depot. + /// + public required ManifestId CurrentManifestId { get; init; } +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs new file mode 100644 index 0000000000..e5be76dcfa --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs @@ -0,0 +1,19 @@ +using NexusMods.Abstractions.Steam.Values; + +namespace NexusMods.Abstractions.Steam.DTOs; + +/// +/// Information about a product (a game) on Steam. +/// +public class ProductInfo +{ + /// + /// The app id of the product. + /// + public AppId AppId { get; init; } + + /// + /// The depots of the product. + /// + public Depot[] Depots { get; init; } +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/IAuthInterventionHandler.cs b/Abstractions/NexusMods.Abstractions.Steam/IAuthInterventionHandler.cs new file mode 100644 index 0000000000..7228b54be5 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/IAuthInterventionHandler.cs @@ -0,0 +1,12 @@ +namespace NexusMods.Abstractions.Steam; + +/// +/// A user intervention handler that can be used to request authorization information from the user. +/// +public interface IAuthInterventionHandler +{ + /// + /// Display a QR code to the user for the given uri. When the token is cancelled, the QR code should be hidden. + /// + public void ShowQRCode(Uri uri, CancellationToken token); +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/IAuthStorage.cs b/Abstractions/NexusMods.Abstractions.Steam/IAuthStorage.cs new file mode 100644 index 0000000000..66ba817b17 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/IAuthStorage.cs @@ -0,0 +1,17 @@ +namespace NexusMods.Abstractions.Steam; + +/// +/// Interface for abstracting away the storage of Steam authentication data. +/// +public interface IAuthStorage +{ + /// + /// Tries to load the authentication data, if it does not exist or fails to load, returns false. + /// + public Task<(bool Success, byte[] Data)> TryLoad(); + + /// + /// Saves the authentication data. + /// + public Task SaveAsync(byte[] data); +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs new file mode 100644 index 0000000000..3f78a7f00b --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs @@ -0,0 +1,15 @@ +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; + +namespace NexusMods.Abstractions.Steam; + +/// +/// An abstraction for a Steam session. +/// +public interface ISteamSession +{ + /// + /// Get the product info for the specified app ID + /// + public Task GetProductInfoAsync(AppId appId, CancellationToken cancellationToken = default); +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj b/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj new file mode 100644 index 0000000000..caa1e85efc --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs new file mode 100644 index 0000000000..8b103e1483 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs @@ -0,0 +1,12 @@ +using TransparentValueObjects; + +namespace NexusMods.Abstractions.Steam.Values; + +/// +/// A globally unique identifier for an application on Steam. +/// +[ValueObject] +public readonly partial struct AppId +{ + +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs new file mode 100644 index 0000000000..9bd3552925 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs @@ -0,0 +1,12 @@ +using TransparentValueObjects; + +namespace NexusMods.Abstractions.Steam.Values; + +/// +/// A globally unique identifier for a depot, a reference to a collection of files on the Steam CDN. +/// +[ValueObject] +public partial struct DepotId +{ + +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs new file mode 100644 index 0000000000..9b7ac84367 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs @@ -0,0 +1,12 @@ +using TransparentValueObjects; + +namespace NexusMods.Abstractions.Steam.Values; + +/// +/// A global unique identifier for a manifest, a specific collection of files that can be downloaded +/// +[ValueObject] +public partial struct ManifestId +{ + +} diff --git a/Directory.Packages.props b/Directory.Packages.props index a62db82c3e..8a86e0937c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,10 +31,12 @@ + + diff --git a/NexusMods.App.sln b/NexusMods.App.sln index c4aa99854a..cd1f6ed3b6 100644 --- a/NexusMods.App.sln +++ b/NexusMods.App.sln @@ -278,6 +278,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.MountAndBla EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Telemetry.Tests", "tests\NexusMods.Telemetry.Tests\NexusMods.Telemetry.Tests.csproj", "{336387F7-3635-43FE-9C23-CBC0CE534989}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Stores", "Stores", "{D5C9FBEA-5BD0-4879-B67B-C728D04206E8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Stores.Steam", "NexusMods.Stores.Steam\NexusMods.Stores.Steam.csproj", "{2987630F-40E8-4A5E-A702-0B7641F345DA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.Steam", "src\Networking\NexusMods.Networking.Steam\NexusMods.Networking.Steam.csproj", "{4A501BBB-389C-460C-B0C3-6F2F968773B1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Steam", "Abstractions\NexusMods.Abstractions.Steam\NexusMods.Abstractions.Steam.csproj", "{24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.Steam.Tests", "tests\Networking\NexusMods.Networking.Steam.Tests\NexusMods.Networking.Steam.Tests.csproj", "{17023DB9-8E31-4397-B3E1-141149987865}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -728,6 +738,22 @@ Global {336387F7-3635-43FE-9C23-CBC0CE534989}.Debug|Any CPU.Build.0 = Debug|Any CPU {336387F7-3635-43FE-9C23-CBC0CE534989}.Release|Any CPU.ActiveCfg = Release|Any CPU {336387F7-3635-43FE-9C23-CBC0CE534989}.Release|Any CPU.Build.0 = Release|Any CPU + {2987630F-40E8-4A5E-A702-0B7641F345DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2987630F-40E8-4A5E-A702-0B7641F345DA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2987630F-40E8-4A5E-A702-0B7641F345DA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2987630F-40E8-4A5E-A702-0B7641F345DA}.Release|Any CPU.Build.0 = Release|Any CPU + {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Release|Any CPU.Build.0 = Release|Any CPU + {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}.Release|Any CPU.Build.0 = Release|Any CPU + {17023DB9-8E31-4397-B3E1-141149987865}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {17023DB9-8E31-4397-B3E1-141149987865}.Debug|Any CPU.Build.0 = Debug|Any CPU + {17023DB9-8E31-4397-B3E1-141149987865}.Release|Any CPU.ActiveCfg = Release|Any CPU + {17023DB9-8E31-4397-B3E1-141149987865}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -857,6 +883,11 @@ Global {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77} = {70D38D24-79AE-4600-8E83-17F3C11BA81F} {336387F7-3635-43FE-9C23-CBC0CE534989} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} + {D5C9FBEA-5BD0-4879-B67B-C728D04206E8} = {E7BAE287-D505-4D6D-A090-665A64309B2D} + {2987630F-40E8-4A5E-A702-0B7641F345DA} = {D5C9FBEA-5BD0-4879-B67B-C728D04206E8} + {4A501BBB-389C-460C-B0C3-6F2F968773B1} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C} + {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} + {17023DB9-8E31-4397-B3E1-141149987865} = {897C4198-884F-448A-B0B0-C2A6D971EAE0} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F9F8352-34DD-42C0-8564-EE9AF34A3501} diff --git a/NexusMods.App.sln.DotSettings b/NexusMods.App.sln.DotSettings index 4ca43b3289..a81d10b1d7 100644 --- a/NexusMods.App.sln.DotSettings +++ b/NexusMods.App.sln.DotSettings @@ -44,6 +44,7 @@ MIME NXM OSX + QR SMAPI VM LG diff --git a/NexusMods.Stores.Steam/CDNPool.cs b/NexusMods.Stores.Steam/CDNPool.cs new file mode 100644 index 0000000000..0975d1a55a --- /dev/null +++ b/NexusMods.Stores.Steam/CDNPool.cs @@ -0,0 +1,9 @@ +namespace NexusMods.Stores.Steam; + +public class CDNPool +{ + public CDNPool() + { + } + +} diff --git a/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj b/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj new file mode 100644 index 0000000000..194da5f628 --- /dev/null +++ b/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj @@ -0,0 +1,20 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + + diff --git a/NexusMods.Stores.Steam/Program.cs b/NexusMods.Stores.Steam/Program.cs new file mode 100644 index 0000000000..555227af4c --- /dev/null +++ b/NexusMods.Stores.Steam/Program.cs @@ -0,0 +1,12 @@ +namespace NexusMods.Stores.Steam; + +public class Program +{ + public static async Task Main(string[] argv) + { + var client = new Session(); + await client.ConnectAsync(); + + return 0; + } +} diff --git a/NexusMods.Stores.Steam/README.md b/NexusMods.Stores.Steam/README.md new file mode 100644 index 0000000000..f10b2d774d --- /dev/null +++ b/NexusMods.Stores.Steam/README.md @@ -0,0 +1,6 @@ +## NexusMods.Stores.Steam + +A minimalist API wrapper for accessing steam files via a Steam Login. For now this is used only +to collect and process hash data about games and their files. If this code ends up working +well enough we may integrate it with the larger app, but for now this is only for supporting +utilities. diff --git a/NexusMods.Stores.Steam/Session.cs b/NexusMods.Stores.Steam/Session.cs new file mode 100644 index 0000000000..d63d3e0170 --- /dev/null +++ b/NexusMods.Stores.Steam/Session.cs @@ -0,0 +1,202 @@ +using System.Collections.Concurrent; +using QRCoder; +using SteamKit2; +using SteamKit2.Authentication; +using SteamKit2.CDN; + +namespace NexusMods.Stores.Steam; + +public class Session +{ + private readonly SteamConfiguration _clientConfiguration; + private readonly SteamClient _steamClient; + private readonly CallbackManager _callbacks; + private readonly SteamUser _steamUser; + private readonly SteamApps _steamApps; + + private ConcurrentBag _licenses = []; + + private bool _running = false; + private readonly SteamContent _steamContent; + private readonly Client _cdnClient; + + public Session() + { + _steamClient = new SteamClient(); + _steamUser = _steamClient.GetHandler()!; + _steamApps = _steamClient.GetHandler()!; + _steamContent = _steamClient.GetHandler()!; + _cdnClient = new Client(_steamClient); + + _callbacks = new CallbackManager(_steamClient); + _callbacks.Subscribe(o => Task.Run(() => ConnectedCallback(o))); + _callbacks.Subscribe(DisconnectedCallback); + _callbacks.Subscribe(LoggedOnCallback); + _callbacks.Subscribe(LicenseListCallback); + _callbacks.Subscribe(o => Task.Run(() => PICSProductInfoCallback(o))); + } + + private Task PICSProductInfoCallback(SteamApps.PICSProductInfoCallback picsProductInfoCallback) + { + var result = new Dictionary(); + foreach (var kv in picsProductInfoCallback.Apps.First().Value.KeyValues.Children) + { + ToJson(result, kv); + } + throw new NotImplementedException(); + } + + private void ToJson(Dictionary result, KeyValue kv) + { + if (kv.Value != null) + result[kv.Name!] = kv.Value; + else + { + var subDict = new Dictionary(); + foreach (var subKv in kv.Children) + { + ToJson(subDict, subKv); + } + result[kv.Name!] = subDict; + } + } + + + public async Task ConnectAsync() + { + _running = true; + _steamClient.Connect(); + while (_running) + { + await _callbacks.RunWaitCallbackAsync(CancellationToken.None); + } + } + + + private void LicenseListCallback(SteamApps.LicenseListCallback obj) + { + foreach (var license in obj.LicenseList) + { + _licenses.Add(license); + } + } + + private void LoggedOnCallback(SteamUser.LoggedOnCallback callback) + { + if (callback.Result != EResult.OK) + { + Console.WriteLine("Unable to logon to Steam: {0} / {1}", callback.Result, callback.ExtendedResult); + _running = false; + return; + } + + Console.WriteLine("Successfully logged on!"); + + Console.WriteLine("Requesting license list..."); + //_steamApps.PICSGetProductInfo(new SteamApps.PICSRequest(413150), null); + + Task.Run(async () => await GetManifestInfo()); + } + + private async Task GetManifestInfo() + { + var appId = (uint)413150; + var depotId = (uint)413151; + var manifestId = 1364246008775303529UL; + var requestCode = await GetDepotManifestRequestCodeAsync(depotId, appId, manifestId, "public"); + var depotKey = await GetDepotKey(depotId, appId); + var servers = await _steamContent.GetServersForSteamPipe(); + var usable = servers + .Where(s => s.Type == "CDN") + .ToArray(); + Random.Shared.Shuffle(usable); + var server = usable.First(); + var cdnAuthToken = await RequestCDNAuthTokenAsync(appId, depotId, server); + + Console.WriteLine("Got {0} servers", servers.Count); + var manifest = await _cdnClient.DownloadManifestAsync(depotId, manifestId, requestCode, server, depotKey, cdnAuthToken: cdnAuthToken); + + Console.WriteLine("Got manifest with {0} files", manifest.Files.Count); + + } + + private async Task RequestCDNAuthTokenAsync(uint appId, uint depotId, Server server) + { + var cdnAuth = await _steamContent.GetCDNAuthToken(appId, depotId, server.Host!); + + if (cdnAuth.Result != EResult.OK) + Console.WriteLine("Failed to get CDN auth token for depot {0}", depotId); + else + Console.WriteLine("Got CDN auth token for depot {0}", depotId); + + return cdnAuth.Token; + } + + private async Task GetDepotKey(uint depotId, uint appId) + { + var key = await _steamApps.GetDepotDecryptionKey(depotId, appId); + if (key.Result != EResult.OK) + Console.WriteLine("Failed to get depot key for depot {0}", depotId); + else + Console.WriteLine("Got depot key for depot {0}", depotId); + + return key!.DepotKey; + } + + private async Task GetDepotManifestRequestCodeAsync(uint depotId, uint appId, ulong manifestId, string branch) + { + var requestCode = await _steamContent.GetManifestRequestCode(depotId, appId, manifestId, branch); + if (requestCode == 0) + Console.WriteLine("Failed to get request code for depot {0} manifest {1}", depotId, manifestId); + else + Console.WriteLine("Got request code {0} for depot {1} manifest {2}", requestCode, depotId, manifestId); + return requestCode; + } + + private void DisconnectedCallback(SteamClient.DisconnectedCallback obj) + { + throw new NotImplementedException(); + } + + private async Task ConnectedCallback(SteamClient.ConnectedCallback callback) + { + var authSession = await _steamClient.Authentication.BeginAuthSessionViaQRAsync(new AuthSessionDetails()); + + authSession.ChallengeURLChanged = () => + { + Console.WriteLine(); + Console.WriteLine("Steam has generated a new QR code for you to scan"); + + DrawQRCode(authSession); + }; + + DrawQRCode(authSession); + var pollResponse = await authSession.PollingWaitForResultAsync(); + + Console.WriteLine("Logging in as " + pollResponse.AccountName + "..."); + _steamUser.LogOn(new SteamUser.LogOnDetails + { + Username = pollResponse.AccountName, + AccessToken = pollResponse.RefreshToken, + } + ); + + } + + private void DrawQRCode(QrAuthSession authSession) + { + Console.WriteLine( $"Challenge URL: {authSession.ChallengeURL}" ); + Console.WriteLine(); + + // Encode the link as a QR code + using var qrGenerator = new QRCodeGenerator(); + var qrCodeData = qrGenerator.CreateQrCode( authSession.ChallengeURL, QRCodeGenerator.ECCLevel.L ); + using var qrCode = new AsciiQRCode( qrCodeData ); + var qrCodeAsAsciiArt = qrCode.GetGraphic( 1, drawQuietZones: false ); + + Console.WriteLine( "Use the Steam Mobile App to sign in via QR code:" ); + Console.WriteLine( qrCodeAsAsciiArt ); + } + + public SteamUser.LogOnDetails LogOnDetails { get; set; } +} diff --git a/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs b/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs new file mode 100644 index 0000000000..8f8b8a0dde --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs @@ -0,0 +1,35 @@ +using NexusMods.Abstractions.Steam; +using NexusMods.Paths; + +namespace NexusMods.Networking.Steam; + + +internal class AppDirectoryAuthStorage(IFileSystem fileSystem) : IAuthStorage +{ + private AbsolutePath _storagePath = fileSystem.GetKnownPath(KnownPath.LocalApplicationDataDirectory) + / (fileSystem.OS.IsOSX ? "NexusMods_App" : "NexusMods.App") + / "steam/auth"; + + public async Task<(bool Success, byte[] Data)> TryLoad() + { + try + { + if (!_storagePath.FileExists) + return (false, []); + + return (true, await _storagePath.ReadAllBytesAsync()); + } + catch + { + return (false, []); + } + + } + + public async Task SaveAsync(byte[] data) + { + if (!_storagePath.Parent.DirectoryExists()) + _storagePath.Parent.CreateDirectory(); + await _storagePath.WriteAllBytesAsync(data); + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/DTOs/AuthData.cs b/src/Networking/NexusMods.Networking.Steam/DTOs/AuthData.cs new file mode 100644 index 0000000000..a70ba4085a --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/DTOs/AuthData.cs @@ -0,0 +1,20 @@ +using System.Text; +using System.Text.Json; + +namespace NexusMods.Networking.Steam.DTOs; + +internal class AuthData +{ + public required string Username { get; init; } + public required string RefreshToken { get; init; } + + public byte[] Save() + { + return Encoding.UTF8.GetBytes(JsonSerializer.Serialize(this)); + } + + public static AuthData Load(byte[] data) + { + return JsonSerializer.Deserialize(Encoding.UTF8.GetString(data))!; + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/LoggingAuthInterventionHandler.cs b/src/Networking/NexusMods.Networking.Steam/LoggingAuthInterventionHandler.cs new file mode 100644 index 0000000000..bb084af465 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/LoggingAuthInterventionHandler.cs @@ -0,0 +1,22 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.Steam; +using QRCoder; + +namespace NexusMods.Networking.Steam; + +public class LoggingAuthInterventionHandler(ILogger logger) : IAuthInterventionHandler +{ + public void ShowQRCode(Uri uri, CancellationToken token) + { + logger.LogInformation("Please scan this QR code with the Steam app on your phone."); + + using var qrGenerator = new QRCodeGenerator(); + using var qrCodeData = qrGenerator.CreateQrCode(uri.ToString(), QRCodeGenerator.ECCLevel.L); + using var qrCode = new AsciiQRCode(qrCodeData); + var asciiArt = qrCode.GetGraphic(1, drawQuietZones: false); + var lines = $"\n\n{asciiArt}\n\n"; + logger.LogInformation(lines); + Debug.WriteLine(lines); + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj new file mode 100644 index 0000000000..35aebc4866 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -0,0 +1,20 @@ + + + + net9.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs new file mode 100644 index 0000000000..426f24ab56 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs @@ -0,0 +1,16 @@ +using NexusMods.Abstractions.Steam.DTOs; +using SteamKit2; + +namespace NexusMods.Networking.Steam; + +/// +/// Parses a PICSProductInfoCallback into a . +/// +public static class ProductInfoParser +{ + public static ProductInfo Parse(SteamApps.PICSProductInfoCallback callback) + { + + return null!; + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/Services.cs b/src/Networking/NexusMods.Networking.Steam/Services.cs new file mode 100644 index 0000000000..c19d84970e --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/Services.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.Steam; + +namespace NexusMods.Networking.Steam; + +public static class Services +{ + /// + /// Add the steam store DI systems to the container + /// + public static IServiceCollection AddSteamStore(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } + + /// + /// Adds a logging authentication handler to the DI container + /// + public static IServiceCollection AddLoggingAuthenticationHandler(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } + + /// + /// Adds auth storage to the DI container that stores the auth data in the app directory + /// + public static IServiceCollection AddLocalAuthFileStorage(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } + +} diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs new file mode 100644 index 0000000000..0e14fa7c04 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -0,0 +1,176 @@ +using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Networking.Steam.DTOs; +using SteamKit2; +using SteamKit2.Authentication; +using SteamKit2.CDN; + +namespace NexusMods.Networking.Steam; + +/// +/// Base class for a Steam session. Steam works in a rather strange way. Internally it communicates +/// via websockets. This means that many of the operations require sending a message, then listening +/// on a separate callback handler for the response. This class attempts to abstract this process and +/// provide a cleaner interface for other parts of the app. +/// +public class Session : ISteamSession +{ + private bool _isConnected = false; + private bool _isLoggedOn = false; + + private readonly ILogger _logger; + private readonly IAuthInterventionHandler _handler; + private readonly SteamConfiguration _steamConfiguration; + + /// + /// Base steam component, this is used for communicating with the Steam network. + /// + private readonly SteamClient _steamClient; + + /// + /// The component used for user related operations. + /// + private readonly SteamUser _steamUser; + + /// + /// The component used for getting information about apps (games) + /// + private readonly SteamApps _steamApps; + + /// + /// Component for getting content information (actual game data) + /// + private readonly SteamContent _steamContent; + + /// + /// CDN data, used to get download locations for game data. + /// + private readonly Client _cdnClient; + + private readonly CallbackManager _callbacks; + private readonly IAuthStorage _authStorage; + + public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage, HttpClient httpClient) + { + + _logger = logger; + _handler = handler; + _authStorage = storage; + + _steamConfiguration = SteamConfiguration.Create(configurator => + { + configurator.WithHttpClientFactory(() => httpClient); + }); + _steamClient = new SteamClient(_steamConfiguration); + _steamUser = _steamClient.GetHandler()!; + _steamApps = _steamClient.GetHandler()!; + _steamContent = _steamClient.GetHandler()!; + _cdnClient = new Client(_steamClient); + + + _callbacks = new CallbackManager(_steamClient); + _callbacks.Subscribe(WrapAsync(ConnectedCallback)); + _callbacks.Subscribe(WrapAsync(DisconnectedCallback)); + _callbacks.Subscribe(WrapAsync(LoggedOnCallback)); + _callbacks.Subscribe(WrapAsync(LicenseListCallback)); + } + + private Task LicenseListCallback(SteamApps.LicenseListCallback arg) + { + return Task.CompletedTask; + } + + private async Task PICSProductInfoCallback(SteamApps.PICSProductInfoCallback callback) + { + _logger.LogInformation("Received PICSProductInfoCallback"); + } + + private async Task LoggedOnCallback(SteamUser.LoggedOnCallback callback) + { + _isLoggedOn = true; + _logger.LogInformation("Logged on to Steam network."); + } + + private async Task DisconnectedCallback(SteamClient.DisconnectedCallback callback) + { + _logger.LogInformation("Disconnected from Steam network."); + } + + private async Task ConnectedCallback(SteamClient.ConnectedCallback callback) + { + _isConnected = true; + var (success, data) = await _authStorage.TryLoad(); + if (success) + { + _logger.LogInformation("Using saved auth data to log in."); + var authData = AuthData.Load(data); + + _steamUser.LogOn(new SteamUser.LogOnDetails + { + Username = authData.Username, + AccessToken = authData.RefreshToken, + } + ); + } + else + { + _logger.LogInformation("No saved auth data, logging in via QR code."); + var authSession = await _steamClient.Authentication.BeginAuthSessionViaQRAsync(new AuthSessionDetails()); + + authSession.ChallengeURLChanged = () => { _handler.ShowQRCode(new Uri(authSession.ChallengeURL, UriKind.Absolute), CancellationToken.None); }; + + _handler.ShowQRCode(new Uri(authSession.ChallengeURL, UriKind.Absolute), CancellationToken.None); + var pollResponse = await authSession.PollingWaitForResultAsync(); + + await _authStorage.SaveAsync(new AuthData + { + Username = pollResponse.AccountName, + RefreshToken = pollResponse.RefreshToken, + }.Save()); + + _steamUser.LogOn(new SteamUser.LogOnDetails + { + Username = pollResponse.AccountName, + AccessToken = pollResponse.RefreshToken, + } + ); + } + } + + /// + /// Wraps an async action in a synchronous action. + /// + private Action WrapAsync(Func action) + { + return arg => Task.Run(async () => await action(arg)); + } + + + public async Task GetProductInfoAsync(AppId appId, CancellationToken cancellationToken = default) + { + await ConnectedAsync(cancellationToken); + var jobs = await _steamApps.PICSGetProductInfo(new SteamApps.PICSRequest(appId.Value), null); + if (jobs.Failed) + throw new Exception("Failed to get product info for app " + appId.Value); + + return ProductInfoParser.Parse(jobs.Results![0]); + } + + /// + /// Performs a login if required, returns once the login is complete + /// + private async Task ConnectedAsync(CancellationToken cancellationToken) + { + if (!_isConnected) + { + _steamClient.Connect(); + } + + while (!_isLoggedOn) + { + await _callbacks.RunWaitCallbackAsync(cancellationToken); + } + } +} diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs new file mode 100644 index 0000000000..6454b19155 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.Values; +using Xunit; + +namespace NexusMods.Networking.Steam.Tests; + +public class BasicApiTests(ILogger logger, ISteamSession session) +{ + private static readonly AppId SdvAppId = AppId.From(413150); + + [Fact] + public async Task CanGetProductInfo() + { + var info = await session.GetProductInfoAsync(SdvAppId); + Assert.NotNull(info); + } + +} diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/NexusMods.Networking.Steam.Tests.csproj b/tests/Networking/NexusMods.Networking.Steam.Tests/NexusMods.Networking.Steam.Tests.csproj new file mode 100644 index 0000000000..8070bd1917 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/NexusMods.Networking.Steam.Tests.csproj @@ -0,0 +1,17 @@ + + + NexusMods.Networking.Steam.Tests + + + + net9.0 + enable + enable + + + + + + + + diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs new file mode 100644 index 0000000000..0d1ccd7eb9 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.DependencyInjection; +using NexusMods.Paths; + +namespace NexusMods.Networking.Steam.Tests; + +public class Startup +{ + public void ConfigureServices(IServiceCollection services) + { + services + .AddSingleton() + .AddSteamStore() + .AddLoggingAuthenticationHandler() + .AddLocalAuthFileStorage() + .AddFileSystem(); + } +} + From 669d794b057ec55650d067cac0b16c71d5997d92 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 11 Dec 2024 12:32:58 -0700 Subject: [PATCH 02/22] Implemented the rest of the Steam flow for getting manifests. Introduced a .Hashes abstraction library for hash value types. Moved Chunked streams into the IO abstractions library --- .../DTOs/Depot.cs | 13 +-- .../DTOs/Manifest.cs | 93 ++++++++++++++++++ .../DTOs/ManifestInfo.cs | 26 +++++ .../DTOs/ProductInfo.cs | 5 + .../ISteamSession.cs | 5 + .../NexusMods.Abstractions.Steam.csproj | 5 + .../Values/DepotId.cs | 2 +- .../Values/ManifestId.cs | 2 +- NexusMods.App.sln | 7 ++ .../NexusMods.Abstractions.Hashes/Crc32.cs | 11 +++ .../NexusMods.Abstractions.Hashes.csproj | 8 ++ .../NexusMods.Abstractions.Hashes/Sha1.cs | 95 +++++++++++++++++++ .../ChunkedStreams/ChunkedStream.cs | 2 +- .../ChunkedStreams/IChunkedStreamSource.cs | 2 +- .../ChunkedStreams/LightweightLRUCache.cs | 2 +- .../AppDirectoryAuthStorage.cs | 6 +- .../NexusMods.Networking.Steam/CDNPool.cs | 74 +++++++++++++++ .../ManifestParser.cs | 40 ++++++++ .../ProductInfoParser.cs | 69 +++++++++++++- .../NexusMods.Networking.Steam/Session.cs | 68 ++++++++++++- src/NexusMods.DataModel/NxFileStore.cs | 2 +- .../BasicApiTests.cs | 8 +- .../ChunkedReaders/ChunkedReaderTests.cs | 2 +- .../LightweightLRUCacheTests.cs | 2 +- 24 files changed, 526 insertions(+), 23 deletions(-) create mode 100644 Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs create mode 100644 Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs rename src/{NexusMods.DataModel => Abstractions/NexusMods.Abstractions.IO}/ChunkedStreams/ChunkedStream.cs (99%) rename src/{NexusMods.DataModel => Abstractions/NexusMods.Abstractions.IO}/ChunkedStreams/IChunkedStreamSource.cs (96%) rename src/{NexusMods.DataModel => Abstractions/NexusMods.Abstractions.IO}/ChunkedStreams/LightweightLRUCache.cs (98%) create mode 100644 src/Networking/NexusMods.Networking.Steam/CDNPool.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/ManifestParser.cs diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs index 26c2e1fe71..dc8004289a 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Depot.cs @@ -8,14 +8,9 @@ namespace NexusMods.Abstractions.Steam.DTOs; public class Depot { /// - /// The name of the depot. + /// The OSes that the depot is available on. /// - public required string Name { get; init; } - - /// - /// The app id associated with the depot. - /// - public required AppId AppId { get; init; } + public required string[] OsList { get; init; } /// /// The id of the depot. @@ -23,7 +18,7 @@ public class Depot public required DepotId DepotId { get; init; } /// - /// The Current ManifestId of the depot. + /// The manifests associated with the depot, with a key for each available branch /// - public required ManifestId CurrentManifestId { get; init; } + public required Dictionary Manifests { get; init; } } diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs new file mode 100644 index 0000000000..ae4a0b983b --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs @@ -0,0 +1,93 @@ +using NexusMods.Abstractions.Hashes; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; + +namespace NexusMods.Abstractions.Steam.DTOs; + +/// +/// A full, detailed manifest for a specific manifest id +/// +public class Manifest +{ + /// + /// The gid of the manifest + /// + public required ManifestId ManifestId { get; init; } + + /// + /// The files in the manifest + /// + public required FileData[] Files { get; init; } + + /// + /// The depot id of the manifest + /// + public required DepotId DepotId { get; init; } + + /// + /// The time the manifest was created + /// + public required DateTimeOffset CreationTime { get; init; } + + /// + /// The size of all files in the manifest, compressed + /// + public required Size TotalCompressedSize { get; init; } + + /// + /// The size of all files in the manifest, uncompressed + /// + public required Size TotalUncompressedSize { get; init; } + + public class FileData + { + /// + /// The name of the file + /// + public RelativePath Path { get; init; } + + /// + /// The size of the file, compressed + /// + public Size Size { get; init; } + + /// + /// The Sha1 hash of the file + /// + public Sha1 Hash { get; init; } + + /// + /// The chunks of the file + /// + public Chunk[] Chunks { get; init; } + } + + public class Chunk + { + /// + /// The id of the chunk + /// + public required Sha1 ChunkId { get; init; } + + /// + /// The crc32 checksum of the chunk + /// + public required Crc32 Checksum { get; init; } + + /// + /// The offset of the chunk in the resulting file + /// + public required ulong Offset { get; init; } + + /// + /// The size of the chunk, compressed + /// + public required Size CompressedSize { get; init; } + + /// + /// The size of the chunk, uncompressed + /// + public required Size UncompressedSize { get; init; } + + } +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs new file mode 100644 index 0000000000..8c48741cb5 --- /dev/null +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs @@ -0,0 +1,26 @@ +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; + +namespace NexusMods.Abstractions.Steam.DTOs; + +/// +/// Meta information about a manifest, not the actual contents, just the id +/// and the size of the files in aggregate. +/// +public class ManifestInfo +{ + /// + /// The globally unique identifier of the manifest. + /// + public required ManifestId ManifestId { get; init; } + + /// + /// The size of the downloaded files + /// + public required Size Size { get; init; } + + /// + /// The size of the files on the CDN + /// + public required Size DownloadSize { get; init; } +} diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs index e5be76dcfa..8d6d6eae25 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs @@ -7,6 +7,11 @@ namespace NexusMods.Abstractions.Steam.DTOs; /// public class ProductInfo { + /// + /// The revision number of this product info. + /// + public required uint ChangeNumber { get; init; } + /// /// The app id of the product. /// diff --git a/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs index 3f78a7f00b..878172e5a6 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs @@ -12,4 +12,9 @@ public interface ISteamSession /// Get the product info for the specified app ID /// public Task GetProductInfoAsync(AppId appId, CancellationToken cancellationToken = default); + + /// + /// Get the manifest data for a specific manifest + /// + public Task GetManifestContents(AppId appId, DepotId depotId, ManifestId manifestId, string branch, CancellationToken token = default); } diff --git a/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj b/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj index caa1e85efc..1e25155a29 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj +++ b/Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj @@ -2,6 +2,11 @@ + + + + + diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs index 9bd3552925..fab365d65d 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs @@ -6,7 +6,7 @@ namespace NexusMods.Abstractions.Steam.Values; /// A globally unique identifier for a depot, a reference to a collection of files on the Steam CDN. /// [ValueObject] -public partial struct DepotId +public readonly partial struct DepotId { } diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs index 9b7ac84367..74b07e2653 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs @@ -6,7 +6,7 @@ namespace NexusMods.Abstractions.Steam.Values; /// A global unique identifier for a manifest, a specific collection of files that can be downloaded /// [ValueObject] -public partial struct ManifestId +public readonly partial struct ManifestId { } diff --git a/NexusMods.App.sln b/NexusMods.App.sln index cd1f6ed3b6..4cfb5f5062 100644 --- a/NexusMods.App.sln +++ b/NexusMods.App.sln @@ -288,6 +288,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Stea EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.Steam.Tests", "tests\Networking\NexusMods.Networking.Steam.Tests\NexusMods.Networking.Steam.Tests.csproj", "{17023DB9-8E31-4397-B3E1-141149987865}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Hashes", "src\Abstractions\NexusMods.Abstractions.Hashes\NexusMods.Abstractions.Hashes.csproj", "{AF703852-D7B0-4BAD-8C75-B6046C6F0490}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -754,6 +756,10 @@ Global {17023DB9-8E31-4397-B3E1-141149987865}.Debug|Any CPU.Build.0 = Debug|Any CPU {17023DB9-8E31-4397-B3E1-141149987865}.Release|Any CPU.ActiveCfg = Release|Any CPU {17023DB9-8E31-4397-B3E1-141149987865}.Release|Any CPU.Build.0 = Release|Any CPU + {AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -888,6 +894,7 @@ Global {4A501BBB-389C-460C-B0C3-6F2F968773B1} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C} {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} {17023DB9-8E31-4397-B3E1-141149987865} = {897C4198-884F-448A-B0B0-C2A6D971EAE0} + {AF703852-D7B0-4BAD-8C75-B6046C6F0490} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F9F8352-34DD-42C0-8564-EE9AF34A3501} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs new file mode 100644 index 0000000000..2dd9aad94e --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs @@ -0,0 +1,11 @@ +using TransparentValueObjects; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// A value representing a 32-bit Cyclic Redundancy Check (CRC) hash. +/// +[ValueObject] +public readonly partial struct Crc32 +{ +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj b/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj new file mode 100644 index 0000000000..0b5a79ba6b --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs new file mode 100644 index 0000000000..1a0431622f --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs @@ -0,0 +1,95 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// A SHA-1 hash, 20 bytes long. +/// +public unsafe struct Sha1 : IEquatable +{ + public bool Equals(Sha1 other) + { + return WritableSpan.SequenceEqual(other.WritableSpan); + } + + public override bool Equals(object? obj) + { + return obj is Sha1 other && Equals(other); + } + + public override int GetHashCode() + { + return MemoryMarshal.Read(WritableSpan); + } + + private fixed byte _value[20]; + + /// + /// Get the hash as a byte span. + /// + public static Sha1 From(ReadOnlySpan value) + { + if (value.Length != 20) + throw new ArgumentException("The value must be 20 bytes long.", nameof(value)); + + var sha = new Sha1(); + value.CopyTo(sha.WritableSpan); + + return sha; + } + + /// + /// Convert the hash to a byte array. + /// + public byte[] ToArray() + { + var array = new byte[20]; + WritableSpan.CopyTo(array); + return array; + } + + /// + /// Allocation free parsing of a SHA-1 hash from a hex string. + /// + public static Sha1 ParseFromHex(string hex) + { + if (hex.Length != 40) + throw new ArgumentException("The hex string must be 40 characters long.", nameof(hex)); + + var sha = new Sha1(); + Convert.FromHexString(hex, sha.WritableSpan, out _, out _); + return sha; + } + + /// + /// Try to convert the hash to a hex string span. + /// + public bool TryToHex(Span hex) + { + Debug.Assert(hex.Length >= 40); + return Convert.TryToHexString(WritableSpan, hex, out _); + } + + /// + /// Convert the hash to a hex string. + /// + public override string ToString() + { + return Convert.ToHexString(WritableSpan); + } + + /// + /// Get a span to the value. + /// + private Span WritableSpan + { + get + { + fixed(byte* pValue = _value) + { + return new(pValue, 20); + } + } + } +} diff --git a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs similarity index 99% rename from src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs rename to src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs index 1abf63a3dc..c32a6ea3ef 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs @@ -2,7 +2,7 @@ using System.Diagnostics; using Reloaded.Memory.Extensions; -namespace NexusMods.DataModel.ChunkedStreams; +namespace NexusMods.Abstractions.IO.ChunkedStreams; /// /// A stream that reads data in chunks, caching the chunks in a cache and allowing diff --git a/src/NexusMods.DataModel/ChunkedStreams/IChunkedStreamSource.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs similarity index 96% rename from src/NexusMods.DataModel/ChunkedStreams/IChunkedStreamSource.cs rename to src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs index 11536d1629..a2c74b33a3 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/IChunkedStreamSource.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs @@ -1,6 +1,6 @@ using NexusMods.Paths; -namespace NexusMods.DataModel.ChunkedStreams; +namespace NexusMods.Abstractions.IO.ChunkedStreams; /// /// A source of data for a . Sizes of chunks should be no larger diff --git a/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/LightweightLRUCache.cs similarity index 98% rename from src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs rename to src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/LightweightLRUCache.cs index fa41ea9c33..d0a7d6ec37 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/LightweightLRUCache.cs @@ -1,6 +1,6 @@ using Reloaded.Memory.Extensions; -namespace NexusMods.DataModel.ChunkedStreams; +namespace NexusMods.Abstractions.IO.ChunkedStreams; /// /// An extremely lightweight LRU cache that is not thread safe. All values are expected diff --git a/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs b/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs index 8f8b8a0dde..3d698307cb 100644 --- a/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs +++ b/src/Networking/NexusMods.Networking.Steam/AppDirectoryAuthStorage.cs @@ -6,9 +6,9 @@ namespace NexusMods.Networking.Steam; internal class AppDirectoryAuthStorage(IFileSystem fileSystem) : IAuthStorage { - private AbsolutePath _storagePath = fileSystem.GetKnownPath(KnownPath.LocalApplicationDataDirectory) - / (fileSystem.OS.IsOSX ? "NexusMods_App" : "NexusMods.App") - / "steam/auth"; + private readonly AbsolutePath _storagePath = fileSystem.GetKnownPath(KnownPath.LocalApplicationDataDirectory) + / (fileSystem.OS.IsOSX ? "NexusMods_App" : "NexusMods.App") + / "steam/auth"; public async Task<(bool Success, byte[] Data)> TryLoad() { diff --git a/src/Networking/NexusMods.Networking.Steam/CDNPool.cs b/src/Networking/NexusMods.Networking.Steam/CDNPool.cs new file mode 100644 index 0000000000..a996de67f7 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/CDNPool.cs @@ -0,0 +1,74 @@ +using System.Collections.Concurrent; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using SteamKit2; +using SteamKit2.CDN; + +namespace NexusMods.Networking.Steam; + +public class CDNPool +{ + private readonly Session _session; + + /// + /// CDN servers that we can use to download content. + /// + private ConcurrentBag _servers = []; + private Server? _currentServer = null; + + /// + /// Cached auth tokens for CDN servers. + /// + private ConcurrentDictionary<(string Host, DepotId DepotId), string> _authTokens = new(); + + public CDNPool(Session session) + { + _session = session; + } + + internal async ValueTask GetServer() + { + if (_servers.IsEmpty) + { + var servers = await _session.Content.GetServersForSteamPipe(); + foreach (var server in servers) + { + if (server.Type != "CDN") + continue; + _servers.Add(server); + } + } + if (_currentServer == null) + _currentServer = _servers.First(); + return _currentServer; + } + + /// + /// Get a CDN auth token for a given depot on a given server. + /// + private async Task GetCDNAuthTokenAsync(AppId appId, DepotId depotId, Server server) + { + // Check if we already have an auth token for this server + if (_authTokens.TryGetValue((server.Host!, depotId), out var token)) + return token; + + var key = await _session.Content.GetCDNAuthToken(appId.Value, depotId.Value, server.Host!); + if (key.Result != EResult.OK) + throw new Exception($"Failed to get CDN auth token for depot {depotId.Value}"); + + _authTokens.TryAdd((server.Host!, depotId), key.Token); + return key.Token; + } + + public async Task GetManifestContents(AppId appId, DepotId depotId, ManifestId manifestId, string branch, CancellationToken token) + { + var requestCode = await _session.GetManifestRequestCodeAsync(appId, depotId, manifestId, branch); + var depotKey = await _session.GetDepotKey(appId, depotId); + var server = await GetServer(); + var cdnAuthToken = await GetCDNAuthTokenAsync(appId, depotId, server); + + var manifest = await _session.CDNClient.DownloadManifestAsync(depotId.Value, manifestId.Value, requestCode, server, depotKey, cdnAuthToken: cdnAuthToken); + var parsed = ManifestParser.Parse(manifest); + return parsed; + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs b/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs new file mode 100644 index 0000000000..153bbc2219 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs @@ -0,0 +1,40 @@ +using NexusMods.Abstractions.Hashes; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; +using SteamKit2; + +namespace NexusMods.Networking.Steam; + +public static class ManifestParser +{ + public static Manifest Parse(DepotManifest manifest) + { + return new Manifest + { + ManifestId = ManifestId.From(manifest.ManifestGID), + TotalCompressedSize = Size.From(manifest.TotalCompressedSize), + TotalUncompressedSize = Size.From(manifest.TotalUncompressedSize), + CreationTime = manifest.CreationTime, + DepotId = DepotId.From(manifest.DepotID), + Files = manifest.Files!.Select(file => new Manifest.FileData + { + Path = file.FileName, + Size = Size.From(file.TotalSize), + Hash = Sha1.From(file.FileHash), + Chunks = file.Chunks.Select(chunk => new Manifest.Chunk + { + ChunkId = Sha1.From(chunk.ChunkID), + Offset = chunk.Offset, + CompressedSize = Size.From(chunk.CompressedLength), + UncompressedSize = Size.From(chunk.UncompressedLength), + Checksum = Crc32.From(chunk.Checksum), + } + ).ToArray(), + } + ).ToArray(), + }; + + } + +} diff --git a/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs index 426f24ab56..e304c34d98 100644 --- a/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs +++ b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs @@ -1,4 +1,7 @@ +using System.Diagnostics.CodeAnalysis; using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; using SteamKit2; namespace NexusMods.Networking.Steam; @@ -10,7 +13,71 @@ public static class ProductInfoParser { public static ProductInfo Parse(SteamApps.PICSProductInfoCallback callback) { + var appInfo = callback.Apps.First(); + var appId = AppId.From(appInfo.Key); + var depotsSection = appInfo.Value.KeyValues.Children.First(kv => kv.Name == "depots").Children; - return null!; + List depots = []; + foreach (var maybeDepot in depotsSection) + { + if (!TryParseDownloadableDepot(appId, maybeDepot, out var depot)) + continue; + depots.Add(depot); + } + + var productInfo = new ProductInfo + { + ChangeNumber = appInfo.Value.ChangeNumber, + AppId = appId, + Depots = depots.ToArray(), + }; + return productInfo; } + + public static bool TryParseDownloadableDepot(AppId appId, KeyValue depot, out Depot result) + { + if (!uint.TryParse(depot.Name, out var parsedDepotId)) + { + result = default(Depot)!; + return false; + } + + var configSection = depot.Children.FirstOrDefault(c => c.Name == "config"); + var osList = configSection?.Children.FirstOrDefault(c => c.Name == "oslist")?.Value ?? ""; + + var depotId = DepotId.From(parsedDepotId); + + var manifestsKey = depot.Children.FirstOrDefault(c => c.Name == "manifests"); + if (manifestsKey == null) + { + result = default(Depot)!; + return false; + } + + Dictionary manifestInfos = new(); + foreach (var branch in manifestsKey.Children) + { + var manifestId = ManifestId.From(ulong.Parse(branch.Children.First(f => f.Name == "gid").Value!)); + var sizeOnDisk = Size.From(ulong.Parse(branch.Children.First(f => f.Name == "size").Value!)); + var downloadSize = Size.From(ulong.Parse(branch.Children.First(f => f.Name == "download").Value!)); + + var manifestInfo = new ManifestInfo + { + ManifestId = manifestId, + Size = sizeOnDisk, + DownloadSize = downloadSize, + }; + manifestInfos.Add(branch.Name!, manifestInfo); + } + + + result = new Depot + { + DepotId = depotId, + OsList = osList.Split(',', ' '), + Manifests = manifestInfos, + }; + return true; + } + } diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs index 0e14fa7c04..d763827435 100644 --- a/src/Networking/NexusMods.Networking.Steam/Session.cs +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -1,3 +1,5 @@ +using System.Collections.Concurrent; +using System.Diagnostics; using Microsoft.Extensions.Logging; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.DTOs; @@ -51,7 +53,11 @@ public class Session : ISteamSession private readonly CallbackManager _callbacks; private readonly IAuthStorage _authStorage; + private readonly CDNPool _cdnPool; + private ConcurrentDictionary<(AppId, DepotId), byte[]> _depotKeys = new(); + private ConcurrentDictionary<(AppId, DepotId, ManifestId, string Branch), ulong> _manifestRequestCodes = new(); + public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage, HttpClient httpClient) { @@ -61,13 +67,14 @@ public Session(ILogger logger, IAuthInterventionHandler handler, IAuthS _steamConfiguration = SteamConfiguration.Create(configurator => { - configurator.WithHttpClientFactory(() => httpClient); + configurator.WithHttpClientFactory(() => new HttpClient()); }); _steamClient = new SteamClient(_steamConfiguration); _steamUser = _steamClient.GetHandler()!; _steamApps = _steamClient.GetHandler()!; _steamContent = _steamClient.GetHandler()!; _cdnClient = new Client(_steamClient); + _cdnPool = new CDNPool(this); _callbacks = new CallbackManager(_steamClient); @@ -77,6 +84,16 @@ public Session(ILogger logger, IAuthInterventionHandler handler, IAuthS _callbacks.Subscribe(WrapAsync(LicenseListCallback)); } + /// + /// The Steam Content module + /// + internal SteamContent Content => _steamContent; + + /// + /// The CDN client, used for downloading game data. + /// + internal Client CDNClient => _cdnClient; + private Task LicenseListCallback(SteamApps.LicenseListCallback arg) { return Task.CompletedTask; @@ -173,4 +190,53 @@ private async Task ConnectedAsync(CancellationToken cancellationToken) await _callbacks.RunWaitCallbackAsync(cancellationToken); } } + + public async Task GetManifestRequestCodeAsync(AppId appId, DepotId depotId, ManifestId manifestId, string branch) + { + if (_manifestRequestCodes.TryGetValue((appId, depotId, manifestId, branch), out var found)) + return found; + await ConnectedAsync(CancellationToken.None); + + var requestCodeResult = await _steamContent.GetManifestRequestCode(depotId.Value, appId.Value, manifestId.Value, branch); + if (requestCodeResult == 0) + { + _logger.LogWarning("Failed to get request code for depot {0} manifest {1}", depotId.Value, manifestId.Value); + throw new Exception("Failed to get request code for depot " + depotId.Value + " manifest " + manifestId.Value); + } + + _logger.LogInformation("Got request code depot {1} manifest {2}", depotId.Value, manifestId.Value); + + _manifestRequestCodes.TryAdd((appId, depotId, manifestId, branch), requestCodeResult); + return requestCodeResult; + } + + /// + /// Get a depot decryption key for a given depot. + /// + public async Task GetDepotKey(AppId appId, DepotId depotId) + { + // Try to get the cached key first + + if (_depotKeys.TryGetValue((appId, depotId), out var keyBytes)) + return keyBytes; + await ConnectedAsync(CancellationToken.None); + + var key = await _steamApps.GetDepotDecryptionKey(depotId.Value, appId.Value); + if (key.Result != EResult.OK) + { + _logger.LogWarning("Failed to get depot key for depot {0}", depotId.Value); + throw new Exception("Failed to get depot key for depot " + depotId.Value); + } + _logger.LogInformation("Got depot key for depot {0}", depotId.Value); + + _depotKeys.TryAdd((appId, depotId), key.DepotKey); + + return key.DepotKey; + } + + public async Task GetManifestContents(AppId appId, DepotId depotId, ManifestId manifestId, string branch, CancellationToken token = default) + { + await ConnectedAsync(token); + return await _cdnPool.GetManifestContents(appId, depotId, manifestId, branch, token); + } } diff --git a/src/NexusMods.DataModel/NxFileStore.cs b/src/NexusMods.DataModel/NxFileStore.cs index 75ebd65c06..c636829e73 100644 --- a/src/NexusMods.DataModel/NxFileStore.cs +++ b/src/NexusMods.DataModel/NxFileStore.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using NexusMods.Abstractions.FileStore.Nx.Models; using NexusMods.Abstractions.IO; +using NexusMods.Abstractions.IO.ChunkedStreams; using NexusMods.Abstractions.MnemonicDB.Attributes; using NexusMods.Abstractions.Settings; using NexusMods.Archives.Nx.FileProviders; @@ -14,7 +15,6 @@ using NexusMods.Archives.Nx.Packing.Unpack; using NexusMods.Archives.Nx.Structs; using NexusMods.Archives.Nx.Utilities; -using NexusMods.DataModel.ChunkedStreams; using NexusMods.Hashing.xxHash3; using NexusMods.MnemonicDB; using NexusMods.MnemonicDB.Abstractions; diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs index 6454b19155..7904cd3e3b 100644 --- a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs @@ -1,3 +1,4 @@ +using FluentAssertions; using Microsoft.Extensions.Logging; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.Values; @@ -13,7 +14,12 @@ public class BasicApiTests(ILogger logger, ISteamSession session) public async Task CanGetProductInfo() { var info = await session.GetProductInfoAsync(SdvAppId); - Assert.NotNull(info); + info.Depots.Should().HaveCountGreaterOrEqualTo(3, "SDV has one depot for each major OS"); + + var depot = info.Depots.First(d => d.OsList.Contains("windows")); + var manifest = await session.GetManifestContents(SdvAppId, depot.DepotId, depot.Manifests["public"].ManifestId, "public"); + + manifest.Should().NotBeNull(); } } diff --git a/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs b/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs index 8061117ef3..b88bf8e0bc 100644 --- a/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs +++ b/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs @@ -1,6 +1,6 @@ using System.Text; using FluentAssertions; -using NexusMods.DataModel.ChunkedStreams; +using NexusMods.Abstractions.IO.ChunkedStreams; using NexusMods.Hashing.xxHash3; using NexusMods.Paths; diff --git a/tests/NexusMods.DataModel.Tests/ChunkedReaders/LightweightLRUCacheTests.cs b/tests/NexusMods.DataModel.Tests/ChunkedReaders/LightweightLRUCacheTests.cs index 25a699c75c..10727819f0 100644 --- a/tests/NexusMods.DataModel.Tests/ChunkedReaders/LightweightLRUCacheTests.cs +++ b/tests/NexusMods.DataModel.Tests/ChunkedReaders/LightweightLRUCacheTests.cs @@ -1,5 +1,5 @@ using FluentAssertions; -using NexusMods.DataModel.ChunkedStreams; +using NexusMods.Abstractions.IO.ChunkedStreams; namespace NexusMods.DataModel.Tests.ChunkedReaders; From f52f6a92a7d3043ccea1f3eac57e32a64f82be4a Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 11 Dec 2024 13:03:28 -0700 Subject: [PATCH 03/22] Boom, can stream files off of Steam --- .../ISteamSession.cs | 6 ++ .../NexusMods.Networking.Steam/CDNPool.cs | 2 +- .../DepotChunkProvider.cs | 56 +++++++++++++++++++ .../NexusMods.Networking.Steam.csproj | 1 + .../NexusMods.Networking.Steam/Session.cs | 15 +++++ .../BasicApiTests.cs | 6 ++ 6 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs diff --git a/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs index 878172e5a6..3541ac55c4 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs @@ -1,5 +1,6 @@ using NexusMods.Abstractions.Steam.DTOs; using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; namespace NexusMods.Abstractions.Steam; @@ -17,4 +18,9 @@ public interface ISteamSession /// Get the manifest data for a specific manifest /// public Task GetManifestContents(AppId appId, DepotId depotId, ManifestId manifestId, string branch, CancellationToken token = default); + + /// + /// Get a readable, seekable, stream for the specified file in the specified manifest + /// + public Stream GetFileStream(AppId appId, Manifest manifest, RelativePath file); } diff --git a/src/Networking/NexusMods.Networking.Steam/CDNPool.cs b/src/Networking/NexusMods.Networking.Steam/CDNPool.cs index a996de67f7..726b66bf0b 100644 --- a/src/Networking/NexusMods.Networking.Steam/CDNPool.cs +++ b/src/Networking/NexusMods.Networking.Steam/CDNPool.cs @@ -46,7 +46,7 @@ internal async ValueTask GetServer() /// /// Get a CDN auth token for a given depot on a given server. /// - private async Task GetCDNAuthTokenAsync(AppId appId, DepotId depotId, Server server) + internal async Task GetCDNAuthTokenAsync(AppId appId, DepotId depotId, Server server) { // Check if we already have an auth token for this server if (_authTokens.TryGetValue((server.Host!, depotId), out var token)) diff --git a/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs b/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs new file mode 100644 index 0000000000..71b3c45029 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs @@ -0,0 +1,56 @@ +using System.Buffers; +using NexusMods.Abstractions.IO.ChunkedStreams; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; +using SteamKit2; + +namespace NexusMods.Networking.Steam; + +public class DepotChunkProvider : IChunkedStreamSource +{ + private readonly Manifest.FileData _fileData; + private readonly Session _session; + private readonly Manifest.Chunk[] _chunksSorted; + private readonly AppId _appId; + private readonly DepotId _depotId; + + public DepotChunkProvider(Session session, AppId appId, DepotId depotId, Manifest manifest, RelativePath relativePath) + { + _appId = appId; + _depotId = depotId; + _fileData = manifest.Files.First(f => f.Path == relativePath); + _chunksSorted = _fileData.Chunks.OrderBy(c => c.Offset).ToArray(); + _session = session; + } + + public Size Size => _fileData.Size; + + /// + /// The Steam CDN provides data in 1MB chunks + /// + public Size ChunkSize => Size.MB; + + public ulong ChunkCount => (ulong)_fileData.Chunks.Length; + + public async Task ReadChunkAsync(Memory buffer, ulong chunkIndex, CancellationToken token = default) + { + var chunk = _chunksSorted[chunkIndex]; + var chunkData = new DepotManifest.ChunkData(chunk.ChunkId.ToArray(), chunk.Checksum.Value, chunk.Offset, + (uint)chunk.CompressedSize.Value, (uint)chunk.UncompressedSize.Value); + var server = await _session.CDNPool.GetServer(); + var depotKey = await _session.GetDepotKey(_appId, _depotId); + var cdnAuthToken = await _session.CDNPool.GetCDNAuthTokenAsync(_appId, _depotId, server); + var rented = ArrayPool.Shared.Rent(buffer.Length); + var read = await _session.CDNClient.DownloadDepotChunkAsync(_depotId.Value, chunkData, server, + rented, depotKey, cdnAuthToken: cdnAuthToken); + if (read != chunkData.UncompressedLength) + throw new InvalidOperationException("Failed to read the entire chunk"); + rented.AsMemory(0, buffer.Length).CopyTo(buffer); + } + + public void ReadChunk(Span buffer, ulong chunkIndex) + { + throw new NotSupportedException(); + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj index 35aebc4866..516ac198ec 100644 --- a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj +++ b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -15,6 +15,7 @@ + diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs index d763827435..5093b5440b 100644 --- a/src/Networking/NexusMods.Networking.Steam/Session.cs +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -1,10 +1,12 @@ using System.Collections.Concurrent; using System.Diagnostics; using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.IO.ChunkedStreams; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.DTOs; using NexusMods.Abstractions.Steam.Values; using NexusMods.Networking.Steam.DTOs; +using NexusMods.Paths; using SteamKit2; using SteamKit2.Authentication; using SteamKit2.CDN; @@ -93,6 +95,11 @@ public Session(ILogger logger, IAuthInterventionHandler handler, IAuthS /// The CDN client, used for downloading game data. /// internal Client CDNClient => _cdnClient; + + /// + /// The CDN pool, used for downloading game data. + /// + internal CDNPool CDNPool => _cdnPool; private Task LicenseListCallback(SteamApps.LicenseListCallback arg) { @@ -239,4 +246,12 @@ public async Task GetManifestContents(AppId appId, DepotId depotId, Ma await ConnectedAsync(token); return await _cdnPool.GetManifestContents(appId, depotId, manifestId, branch, token); } + + public Stream GetFileStream(AppId appId, Manifest manifest, RelativePath file) + { + var chunkedProvider = new DepotChunkProvider(this, appId, manifest.DepotId, + manifest, file + ); + return new ChunkedStream(chunkedProvider); + } } diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs index 7904cd3e3b..6bb5c4d778 100644 --- a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.Values; +using NexusMods.Hashing.xxHash3; using Xunit; namespace NexusMods.Networking.Steam.Tests; @@ -19,6 +20,11 @@ public async Task CanGetProductInfo() var depot = info.Depots.First(d => d.OsList.Contains("windows")); var manifest = await session.GetManifestContents(SdvAppId, depot.DepotId, depot.Manifests["public"].ManifestId, "public"); + var largestFile = manifest.Files.OrderByDescending(f => f.Size).First(); + await using var stream = session.GetFileStream(SdvAppId, manifest, largestFile.Path); + var hash = await stream.xxHash3Async(); + stream.Length.Should().Be((long)largestFile.Size.Value); + manifest.Should().NotBeNull(); } From c6d77030c96e6f1cdecaed1d30ad3bdc461c1f10 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 11 Dec 2024 22:22:31 -0700 Subject: [PATCH 04/22] Can download files, serialize manifests, all that good stuff --- .../Values/AppId.cs | 2 +- .../Values/DepotId.cs | 2 +- .../Values/ManifestId.cs | 2 +- NexusMods.Stores.Steam/Session.cs | 4 +- .../NexusMods.Abstractions.Cli/Renderable.cs | 5 + .../NexusMods.Abstractions.Hashes/Crc32.cs | 28 ++++ .../HashJsonConverter.cs | 28 ++++ .../NexusMods.Abstractions.Hashes/Md5.cs | 113 +++++++++++++++ .../MultiHash.cs | 45 ++++++ .../MultiHasher.cs | 117 ++++++++++++++++ .../NexusMods.Abstractions.Hashes.csproj | 3 + .../NexusMods.Abstractions.Hashes/Sha1.cs | 18 +++ .../ChunkedStreams/ChunkedStream.cs | 10 +- ...exusMods.Abstractions.Serialization.csproj | 1 + .../Services.cs | 2 + .../CLI/RenderingAuthenticationHandler.cs | 25 ++++ .../NexusMods.Networking.Steam/CLI/Verbs.cs | 131 ++++++++++++++++++ .../NexusMods.Networking.Steam.csproj | 2 + .../NexusMods.Networking.Steam/Services.cs | 21 ++- src/NexusMods.App/NexusMods.App.csproj | 1 + src/NexusMods.App/Services.cs | 4 +- .../IRenderer.cs | 1 + .../Startup.cs | 7 +- 23 files changed, 560 insertions(+), 12 deletions(-) create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/Md5.cs create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/MultiHash.cs create mode 100644 src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/CLI/RenderingAuthenticationHandler.cs create mode 100644 src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs index 8b103e1483..0ce3db3f7d 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/AppId.cs @@ -6,7 +6,7 @@ namespace NexusMods.Abstractions.Steam.Values; /// A globally unique identifier for an application on Steam. /// [ValueObject] -public readonly partial struct AppId +public readonly partial struct AppId : IAugmentWith { } diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs index fab365d65d..232983051f 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/DepotId.cs @@ -6,7 +6,7 @@ namespace NexusMods.Abstractions.Steam.Values; /// A globally unique identifier for a depot, a reference to a collection of files on the Steam CDN. /// [ValueObject] -public readonly partial struct DepotId +public readonly partial struct DepotId : IAugmentWith { } diff --git a/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs index 74b07e2653..67e4f4c59a 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/Values/ManifestId.cs @@ -6,7 +6,7 @@ namespace NexusMods.Abstractions.Steam.Values; /// A global unique identifier for a manifest, a specific collection of files that can be downloaded /// [ValueObject] -public readonly partial struct ManifestId +public readonly partial struct ManifestId : IAugmentWith { } diff --git a/NexusMods.Stores.Steam/Session.cs b/NexusMods.Stores.Steam/Session.cs index d63d3e0170..125bda0521 100644 --- a/NexusMods.Stores.Steam/Session.cs +++ b/NexusMods.Stores.Steam/Session.cs @@ -147,9 +147,7 @@ private async Task GetDepotManifestRequestCodeAsync(uint depotId, uint ap { var requestCode = await _steamContent.GetManifestRequestCode(depotId, appId, manifestId, branch); if (requestCode == 0) - Console.WriteLine("Failed to get request code for depot {0} manifest {1}", depotId, manifestId); - else - Console.WriteLine("Got request code {0} for depot {1} manifest {2}", requestCode, depotId, manifestId); + throw new Exception("Unable to get request code for depot " + depotId + " manifest " + manifestId); return requestCode; } diff --git a/src/Abstractions/NexusMods.Abstractions.Cli/Renderable.cs b/src/Abstractions/NexusMods.Abstractions.Cli/Renderable.cs index f58ed2e5cb..a677bb9ba5 100644 --- a/src/Abstractions/NexusMods.Abstractions.Cli/Renderable.cs +++ b/src/Abstractions/NexusMods.Abstractions.Cli/Renderable.cs @@ -20,5 +20,10 @@ public static class Renderable /// /// public static Text Text(string template, string[] args) => new Text { Template = template, Arguments = args}; + + /// + /// Creates a new renderable with a new line at the end. + /// + public static Text TextLine(string template) => new Text { Template = template + "\n" }; } diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs index 2dd9aad94e..3aa1a6ae84 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs @@ -1,3 +1,7 @@ +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using NexusMods.Hashing.xxHash3; using TransparentValueObjects; namespace NexusMods.Abstractions.Hashes; @@ -5,7 +9,31 @@ namespace NexusMods.Abstractions.Hashes; /// /// A value representing a 32-bit Cyclic Redundancy Check (CRC) hash. /// +[JsonConverter(typeof(Crc32JsonConverter))] [ValueObject] public readonly partial struct Crc32 { } + +/// +/// Custom JSON converter for . We're not using augments here as we want hex strings not +/// the raw base 10 value. +/// +internal class Crc32JsonConverter : JsonConverter +{ + public override Crc32 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + Span chars = stackalloc byte[4]; + Convert.FromHexString(reader.GetString()!, chars, out _, out _); + return Crc32.From(MemoryMarshal.Read(chars)); + } + + public override void Write(Utf8JsonWriter writer, Crc32 value, JsonSerializerOptions options) + { + Span span = stackalloc char[8]; + Span bytes = stackalloc byte[4]; + MemoryMarshal.Write(bytes, value.Value); + Convert.TryToHexString(bytes, span, out _); + writer.WriteStringValue(span); + } +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs new file mode 100644 index 0000000000..612586befb --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs @@ -0,0 +1,28 @@ +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using NexusMods.Hashing.xxHash3; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// Json Converter for . +/// +public class HashJsonConverter : JsonConverter +{ + public override Hash Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + Span chars = stackalloc byte[8]; + Convert.FromHexString(reader.GetString()!, chars, out _, out _); + return Hash.FromULong(MemoryMarshal.Read(chars)); + } + + public override void Write(Utf8JsonWriter writer, Hash value, JsonSerializerOptions options) + { + Span span = stackalloc char[16]; + Span bytes = stackalloc byte[8]; + MemoryMarshal.Write(bytes, value.Value); + Convert.TryToHexString(bytes, span, out _); + writer.WriteStringValue(span); + } +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Md5.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Md5.cs new file mode 100644 index 0000000000..1b70f964bb --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Md5.cs @@ -0,0 +1,113 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// A MD5 hash, 16 bytes long. +/// +[JsonConverter(typeof(Md5JsonConverter))] +public unsafe struct Md5 : IEquatable +{ + public bool Equals(Md5 other) + { + return WritableSpan.SequenceEqual(other.WritableSpan); + } + + public override bool Equals(object? obj) + { + return obj is Sha1 other && Equals(other); + } + + public override int GetHashCode() + { + return MemoryMarshal.Read(WritableSpan); + } + + private fixed byte _value[16]; + + /// + /// Get the hash as a byte span. + /// + public static Md5 From(ReadOnlySpan value) + { + if (value.Length != 16) + throw new ArgumentException("The value must be 16 bytes long.", nameof(value)); + + var md5 = new Md5(); + value.CopyTo(md5.WritableSpan); + + return md5; + } + + /// + /// Convert the hash to a byte array. + /// + public byte[] ToArray() + { + var array = new byte[16]; + WritableSpan.CopyTo(array); + return array; + } + + /// + /// Allocation free parsing of a SHA-1 hash from a hex string. + /// + public static Md5 ParseFromHex(string hex) + { + if (hex.Length != 32) + throw new ArgumentException("The hex string must be 40 characters long.", nameof(hex)); + + var sha = new Md5(); + Convert.FromHexString(hex, sha.WritableSpan, out _, out _); + return sha; + } + + /// + /// Try to convert the hash to a hex string span. + /// + public bool TryToHex(Span hex) + { + Debug.Assert(hex.Length >= 16); + return Convert.TryToHexString(WritableSpan, hex, out _); + } + + /// + /// Convert the hash to a hex string. + /// + public override string ToString() + { + return Convert.ToHexString(WritableSpan); + } + + /// + /// Get a span to the value. + /// + private Span WritableSpan + { + get + { + fixed(byte* pValue = _value) + { + return new(pValue, 16); + } + } + } +} + +internal class Md5JsonConverter : JsonConverter +{ + public override Md5 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return Md5.ParseFromHex(reader.GetString()!); + } + + public override void Write(Utf8JsonWriter writer, Md5 value, JsonSerializerOptions options) + { + Span hex = stackalloc char[32]; + value.TryToHex(hex); + writer.WriteStringValue(hex); + } +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHash.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHash.cs new file mode 100644 index 0000000000..50cd0ecc4d --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHash.cs @@ -0,0 +1,45 @@ +using NexusMods.Hashing.xxHash3; +using NexusMods.Paths; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// A grouping of multiple hashes for a single piece of data. +/// +public record MultiHash +{ + /// + /// The xxHash3 hash of the data. + /// + public required Hash XxHash3 { get; init; } + + /// + /// The xxHash64 hash of the data. + /// + public required Hash XxHash64 { get; init; } + + /// + /// the minimal hash of the data. + /// + public required Hash MinimalHash { get; init; } + + /// + /// The Sha1 hash of the data. + /// + public required Sha1 Sha1 { get; init; } + + /// + /// The Md5 hash of the data. + /// + public required Md5 Md5 { get; init; } + + /// + /// The Crc32 hash of the data. + /// + public required Crc32 Crc32 { get; init; } + + /// + /// The size of the data in bytes. + /// + public required Size Size { get; init; } +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs new file mode 100644 index 0000000000..c115366381 --- /dev/null +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs @@ -0,0 +1,117 @@ +using System.Buffers; +using System.IO.Hashing; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using NexusMods.Hashing.xxHash3; +using NexusMods.Paths; + +namespace NexusMods.Abstractions.Hashes; + +/// +/// Hashes to all the common hash algorithms. +/// +public class MultiHasher +{ + private readonly MD5 _md5; + private readonly XxHash3 _xxHash3; + private readonly System.IO.Hashing.Crc32 _crc32; + private readonly SHA1 _sha1; + private readonly byte[] _buffer; + private readonly XxHash3 _minimalHash; + private readonly XxHash64 _xxHash64; + + private const int BufferSize = 64 * 1024; + private const int MaxFullFileHash = BufferSize * 2; + + /// + /// Default constructor. + /// + public MultiHasher() + { + _md5 = MD5.Create(); + _xxHash3 = new XxHash3(); + _xxHash64 = new XxHash64(); + _minimalHash = new XxHash3(); + _crc32 = new System.IO.Hashing.Crc32(); + _sha1 = SHA1.Create(); + _buffer = new byte[4096]; + } + + /// + /// Hashes the stream to all the common hash algorithms. + /// + public async Task HashStream(Stream stream, CancellationToken token) + { + stream.Position = 0; + + while (true) + { + token.ThrowIfCancellationRequested(); + var read = await stream.ReadAsync(_buffer, 0, _buffer.Length, token); + if (read == 0) + break; + var span = _buffer.AsSpan(0, read); + _xxHash3.Append(span); + _xxHash64.Append(span); + _crc32.Append(span); + _md5.TransformBlock(_buffer, 0, read, _buffer, 0); + _sha1.TransformBlock(_buffer, 0, read, _buffer, 0); + } + + _md5.TransformFinalBlock(_buffer, 0, 0); + _sha1.TransformFinalBlock(_buffer, 0, 0); + await MinimalHash(_minimalHash, stream, token); + + var result = new MultiHash + { + XxHash3 = Hash.From(_xxHash3.GetCurrentHashAsUInt64()), + XxHash64 = Hash.From(_xxHash64.GetCurrentHashAsUInt64()), + MinimalHash = Hash.From(_minimalHash.GetCurrentHashAsUInt64()), + Sha1 = Sha1.From(_sha1.Hash), + Md5 = Md5.From(_md5.Hash), + Size = Size.FromLong(stream.Length), + Crc32 = Crc32.From(_crc32.GetCurrentHashAsUInt32()), + }; + + + return result; + } + + /// + /// Calculates a minimal hash of the stream. + /// + public static async Task MinimalHash(XxHash3 hasher, Stream stream, CancellationToken cancellationToken = default) + { + stream.Position = 0; + if (stream.Length <= MaxFullFileHash) + { + await hasher.AppendAsync(stream, cancellationToken); + return; + } + + using var bufferOwner = MemoryPool.Shared.Rent(BufferSize); + var buffer = bufferOwner.Memory[..BufferSize]; + + // Read the block at the start of the file + await stream.ReadExactlyAsync(buffer, cancellationToken); + hasher.Append(buffer.Span); + + // Read the block at the end of the file + stream.Position = stream.Length - BufferSize; + await stream.ReadExactlyAsync(buffer, cancellationToken); + hasher.Append(buffer.Span); + + // Read the block in the middle, if the file is too small, offset the middle enough to not read past the end + // of the file + var middleOffset = Math.Min(stream.Length / 2, stream.Length - BufferSize); + stream.Position = middleOffset; + await stream.ReadExactlyAsync(buffer, cancellationToken); + hasher.Append(buffer.Span); + + // Add the length of the file to the hash (as a ulong) + Span lengthBuffer = stackalloc byte[sizeof(long)]; + MemoryMarshal.Write(lengthBuffer, (ulong)stream.Length); + hasher.Append(lengthBuffer); + } + +} diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj b/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj index 0b5a79ba6b..2badc2e43a 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/NexusMods.Abstractions.Hashes.csproj @@ -3,6 +3,9 @@ + + + diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs index 1a0431622f..1bfa1c65b0 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Sha1.cs @@ -1,11 +1,14 @@ using System.Diagnostics; using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; namespace NexusMods.Abstractions.Hashes; /// /// A SHA-1 hash, 20 bytes long. /// +[JsonConverter(typeof(Sha1JsonConverter))] public unsafe struct Sha1 : IEquatable { public bool Equals(Sha1 other) @@ -93,3 +96,18 @@ private Span WritableSpan } } } + +internal class Sha1JsonConverter : JsonConverter +{ + public override Sha1 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return Sha1.ParseFromHex(reader.GetString()!); + } + + public override void Write(Utf8JsonWriter writer, Sha1 value, JsonSerializerOptions options) + { + Span hex = stackalloc char[40]; + value.TryToHex(hex); + writer.WriteStringValue(hex); + } +} diff --git a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs index c32a6ea3ef..da3e596c4b 100644 --- a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs @@ -30,7 +30,7 @@ public ChunkedStream(T source, int capacity = 16) /// public override void Flush() { } - + /// public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); @@ -69,7 +69,13 @@ public override int Read(Span buffer) return toRead; } - + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var memory = new Memory(buffer, offset, count); + return ReadAsync(memory, cancellationToken).AsTask(); + } + /// public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = new()) diff --git a/src/Abstractions/NexusMods.Abstractions.Serialization/NexusMods.Abstractions.Serialization.csproj b/src/Abstractions/NexusMods.Abstractions.Serialization/NexusMods.Abstractions.Serialization.csproj index 4e41bb62ce..7dfd077a11 100644 --- a/src/Abstractions/NexusMods.Abstractions.Serialization/NexusMods.Abstractions.Serialization.csproj +++ b/src/Abstractions/NexusMods.Abstractions.Serialization/NexusMods.Abstractions.Serialization.csproj @@ -9,6 +9,7 @@ + diff --git a/src/Abstractions/NexusMods.Abstractions.Serialization/Services.cs b/src/Abstractions/NexusMods.Abstractions.Serialization/Services.cs index f251863c1e..319d9f79b5 100644 --- a/src/Abstractions/NexusMods.Abstractions.Serialization/Services.cs +++ b/src/Abstractions/NexusMods.Abstractions.Serialization/Services.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.Hashes; using NexusMods.Abstractions.Serialization.Json; using NexusMods.Abstractions.Settings; @@ -25,6 +26,7 @@ public static IServiceCollection AddSerializationAbstractions(this IServiceColle { var opts = new JsonSerializerOptions(); opts.Converters.Add(new JsonStringEnumConverter()); + opts.Converters.Add(new HashJsonConverter()); foreach (var converter in s.GetServices()) opts.Converters.Add(converter); return opts; diff --git a/src/Networking/NexusMods.Networking.Steam/CLI/RenderingAuthenticationHandler.cs b/src/Networking/NexusMods.Networking.Steam/CLI/RenderingAuthenticationHandler.cs new file mode 100644 index 0000000000..d03bfa0730 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/CLI/RenderingAuthenticationHandler.cs @@ -0,0 +1,25 @@ +using System.Diagnostics; +using NexusMods.Abstractions.Cli; +using NexusMods.Abstractions.Steam; +using NexusMods.ProxyConsole.Abstractions; +using QRCoder; + +namespace NexusMods.Networking.Steam.CLI; + +public class RenderingAuthenticationHandler(IRenderer renderer): IAuthInterventionHandler +{ + public void ShowQRCode(Uri uri, CancellationToken token) + { + _ = Task.Run(async () => + { + await renderer.RenderAsync(Renderable.Text("Please scan the QR code with your Steam Mobile App to continue the authentication process.")); + using var qrGenerator = new QRCodeGenerator(); + using var qrCodeData = qrGenerator.CreateQrCode(uri.ToString(), QRCodeGenerator.ECCLevel.L); + using var qrCode = new AsciiQRCode(qrCodeData); + var asciiArt = qrCode.GetGraphic(1, drawQuietZones: false); + var lines = $"\n\n{asciiArt}\n\n"; + await renderer.RenderAsync(Renderable.Text(lines)); + } + ); + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs b/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs new file mode 100644 index 0000000000..f8881f7f04 --- /dev/null +++ b/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs @@ -0,0 +1,131 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.Cli; +using NexusMods.Abstractions.Hashes; +using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Paths; +using NexusMods.Paths.Extensions; +using NexusMods.ProxyConsole.Abstractions; +using NexusMods.ProxyConsole.Abstractions.VerbDefinitions; + +namespace NexusMods.Networking.Steam.CLI; + +public static class Verbs +{ + internal static IServiceCollection AddSteamVerbs(this IServiceCollection collection) => + collection + .AddVerb(() => IndexSteamApp); + + [Verb("index-steam-app", "Indexes a Steam app and updates the given output folder")] + private static async Task IndexSteamApp( + [Injected] IRenderer renderer, + [Injected] JsonSerializerOptions jsonSerializerOptions, + [Injected] ISteamSession steamSession, + [Option("a", "appId", "The Steam app ID to index")] AppId appId, + [Option("o", "output", "The output folder to write the index to")] AbsolutePath output, + [Injected] CancellationToken token) + { + var indentedOptions = new JsonSerializerOptions(jsonSerializerOptions) + { + WriteIndented = true, + }; + var productInfo = await steamSession.GetProductInfoAsync(appId, token); + + var hashFolder = output / "hashes"; + hashFolder.CreateDirectory(); + + var existingHashes = await LoadExistingHashes(hashFolder, indentedOptions, token); + + // Write the product info to a file + var productFile = output / "stores" / "steam" / "apps" / (productInfo.AppId + ".json").ToRelativePath(); + { + productFile.Parent.CreateDirectory(); + await using var outputStream = productFile.Create(); + await JsonSerializer.SerializeAsync(outputStream, productInfo, indentedOptions, token); + } + + //await renderer.RenderAsync(Renderable.TextLine("Product info written to: " + productFile)); + foreach (var depot in productInfo.Depots) + { + //await renderer.RenderAsync(Renderable.TextLine("Depot: " + depot.DepotId)); + foreach (var (branch, manifestInfo) in depot.Manifests) + { + //await renderer.RenderAsync(Renderable.TextLine("Branch: " + branch)); + //await renderer.RenderAsync(Renderable.TextLine("Manifest: " + manifestInfo.ManifestId)); + + var manifest = await steamSession.GetManifestContents(appId, depot.DepotId, manifestInfo.ManifestId, branch, token); + + var manifestPath = output / "stores" / "steam" / "manifests" / (manifest.ManifestId + ".json").ToRelativePath(); + { + manifestPath.Parent.CreateDirectory(); + await using var outputStream = manifestPath.Create(); + await JsonSerializer.SerializeAsync(outputStream, manifest, indentedOptions, token); + } + + await IndexManifest(steamSession, renderer, appId, output, manifest, indentedOptions, existingHashes, token); + } + } + + return 0; + } + + private static async Task> LoadExistingHashes(AbsolutePath folder, JsonSerializerOptions options, CancellationToken token) + { + var bag = new ConcurrentBag(); + var hashFiles = folder.EnumerateFiles("*.json", true); + + await Parallel.ForEachAsync(hashFiles, token, async (file, token) => + { + await using var stream = file.Read(); + var hash = await JsonSerializer.DeserializeAsync(stream, options, token); + bag.Add(hash!.Sha1); + }); + + return bag; + } + + private static async Task IndexManifest(ISteamSession session, IRenderer renderer, AppId appId, AbsolutePath output, Manifest manifest, JsonSerializerOptions indentedOptions, ConcurrentBag existingHashes, CancellationToken token) + { + var writeLock = new SemaphoreSlim(1, 1); + await Parallel.ForEachAsync(manifest.Files, token, async (file, token) => + { + if (file.Size == Size.Zero) + return; + if (existingHashes.Contains(file.Hash)) + return; + + + await renderer.RenderAsync(Renderable.TextLine("File: " + file.Path)); + await using var stream = session.GetFileStream(appId, manifest, file.Path); + MultiHasher hasher = new(); + var multiHash = await hasher.HashStream(stream, token); + + var fileName = multiHash.XxHash3 + ".json"; + var path = output / "hashes" / fileName[2..4] / fileName[2..]; + + await writeLock.WaitAsync(token); + if (!multiHash.Sha1.Equals(file.Hash)) + throw new InvalidOperationException("Hash mismatch on downloaded file, expected: " + file.Hash + " got: " + multiHash.Sha1); + + try + { + path.Parent.CreateDirectory(); + { + await using var outputStream = path.Create(); + await JsonSerializer.SerializeAsync(outputStream, multiHash, indentedOptions, + token + ); + } + existingHashes.Add(multiHash.Sha1); + } + finally + { + writeLock.Release(); + } + } + ); + } +} diff --git a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj index 516ac198ec..0122d23079 100644 --- a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj +++ b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -15,7 +15,9 @@ + + diff --git a/src/Networking/NexusMods.Networking.Steam/Services.cs b/src/Networking/NexusMods.Networking.Steam/Services.cs index c19d84970e..284757f595 100644 --- a/src/Networking/NexusMods.Networking.Steam/Services.cs +++ b/src/Networking/NexusMods.Networking.Steam/Services.cs @@ -1,5 +1,8 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Networking.Steam.CLI; +using NexusMods.ProxyConsole.Abstractions.VerbDefinitions; namespace NexusMods.Networking.Steam; @@ -8,7 +11,7 @@ public static class Services /// /// Add the steam store DI systems to the container /// - public static IServiceCollection AddSteamStore(this IServiceCollection services) + public static IServiceCollection AddSteam(this IServiceCollection services) { services.AddSingleton(); return services; @@ -32,4 +35,20 @@ public static IServiceCollection AddLocalAuthFileStorage(this IServiceCollection return services; } + public static IServiceCollection AddSteamCli(this IServiceCollection services) + { + services.AddOptionParser(s => + { + if (uint.TryParse(s, out var parsed)) + return (AppId.From(parsed), null); + return (default(AppId), "Invalid AppId"); + } + ); + services.AddSteam(); + services.AddSingleton(); + services.AddLocalAuthFileStorage(); + services.AddSteamVerbs(); + return services; + } + } diff --git a/src/NexusMods.App/NexusMods.App.csproj b/src/NexusMods.App/NexusMods.App.csproj index 2a9ba33017..73d4c58028 100644 --- a/src/NexusMods.App/NexusMods.App.csproj +++ b/src/NexusMods.App/NexusMods.App.csproj @@ -21,6 +21,7 @@ + diff --git a/src/NexusMods.App/Services.cs b/src/NexusMods.App/Services.cs index d4afb92bc7..422453fafc 100644 --- a/src/NexusMods.App/Services.cs +++ b/src/NexusMods.App/Services.cs @@ -25,6 +25,7 @@ using NexusMods.Networking.Downloaders; using NexusMods.Networking.HttpDownloader; using NexusMods.Networking.NexusWebApi; +using NexusMods.Networking.Steam; using NexusMods.Paths; using NexusMods.ProxyConsole; using NexusMods.ProxyConsole.Abstractions.VerbDefinitions; @@ -95,7 +96,8 @@ public static IServiceCollection AddApp(this IServiceCollection services, .AddSingleton() .AddFileSystem() .AddDownloaders() - .AddCleanupVerbs(); + .AddCleanupVerbs() + .AddSteamCli(); if (!startupMode.IsAvaloniaDesigner) services.AddSingleProcess(Mode.Main); diff --git a/src/NexusMods.ProxyConsole.Abstractions/IRenderer.cs b/src/NexusMods.ProxyConsole.Abstractions/IRenderer.cs index 89df1e73a8..159c9908e0 100644 --- a/src/NexusMods.ProxyConsole.Abstractions/IRenderer.cs +++ b/src/NexusMods.ProxyConsole.Abstractions/IRenderer.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using NexusMods.ProxyConsole.Abstractions.Implementations; namespace NexusMods.ProxyConsole.Abstractions; diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs index 0d1ccd7eb9..22ec5e1851 100644 --- a/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/Startup.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using NexusMods.Paths; +using Xunit.DependencyInjection.Logging; namespace NexusMods.Networking.Steam.Tests; @@ -9,10 +11,11 @@ public void ConfigureServices(IServiceCollection services) { services .AddSingleton() - .AddSteamStore() + .AddSteam() .AddLoggingAuthenticationHandler() .AddLocalAuthFileStorage() - .AddFileSystem(); + .AddFileSystem() + .AddLogging(builder => builder.AddXunitOutput().SetMinimumLevel(LogLevel.Trace)); } } From 49b3d9cc43d666bd2b56fe8e51c4eb08374835a9 Mon Sep 17 00:00:00 2001 From: halgari Date: Wed, 11 Dec 2024 23:11:20 -0700 Subject: [PATCH 05/22] Fix the chunked stream so that it can use arbitrarily sized chunks --- .../MultiHasher.cs | 2 +- .../ChunkedStreams/ChunkedStream.cs | 83 +++++++++++-------- .../ChunkedStreams/IChunkedStreamSource.cs | 25 +++--- .../DepotChunkProvider.cs | 11 ++- src/NexusMods.DataModel/NxFileStore.cs | 10 +++ .../BasicApiTests.cs | 8 +- .../ChunkedReaders/ChunkedReaderTests.cs | 40 +++++++-- 7 files changed, 115 insertions(+), 64 deletions(-) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs index c115366381..290cf2a75a 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs @@ -34,7 +34,7 @@ public MultiHasher() _minimalHash = new XxHash3(); _crc32 = new System.IO.Hashing.Crc32(); _sha1 = SHA1.Create(); - _buffer = new byte[4096]; + _buffer = new byte[1024 * 128]; } /// diff --git a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs index da3e596c4b..ed475d5e5f 100644 --- a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs @@ -42,27 +42,18 @@ public override int Read(Span buffer) return 0; } - var chunkIdx = _position / _source.ChunkSize.Value; - var chunkOffset = _position % _source.ChunkSize.Value; - var isLastChunk = chunkIdx == _source.ChunkCount - 1; - var chunk = GetChunk(chunkIdx); + var chunkIdx = FindChunkIndex(_position); + var chunkOffset = _position - _source.GetOffset(chunkIdx); + var chunkSize = _source.GetChunkSize(chunkIdx); + var chunk = GetChunk(chunkIdx)[..chunkSize]; var readToEnd = Math.Clamp(_source.Size.Value - _position, 0, Int32.MaxValue); - var toRead = Math.Min(buffer.Length, (int)(_source.ChunkSize.Value - chunkOffset)); + var toRead = Math.Min(buffer.Length, chunk.Length - (int)chunkOffset); toRead = Math.Min(toRead, (int)readToEnd); - - if (isLastChunk) - { - var lastChunkExtraSize = _source.Size.Value % _source.ChunkSize.Value; - if (lastChunkExtraSize > 0) - { - toRead = Math.Min(toRead, (int)lastChunkExtraSize); - } - } chunk.Slice((int)chunkOffset, toRead) .Span - .CopyTo(buffer.SliceFast(0, toRead)); + .CopyTo(buffer); _position += (ulong)toRead; Debug.Assert(_position <= _source.Size.Value, "Read more than the size of the stream"); @@ -85,26 +76,18 @@ public override async ValueTask ReadAsync(Memory buffer, return 0; } - var chunkIdx = _position / _source.ChunkSize.Value; - var chunkOffset = _position % _source.ChunkSize.Value; - var isLastChunk = chunkIdx == _source.ChunkCount - 1; - var chunk = await GetChunkAsync(chunkIdx, cancellationToken); + var chunkIdx = FindChunkIndex(_position); + var chunkOffset = _position - _source.GetOffset(chunkIdx); + var chunkSize = _source.GetChunkSize(chunkIdx); + var chunk = (await GetChunkAsync(chunkIdx, cancellationToken))[..chunkSize]; var readToEnd = Math.Clamp(_source.Size.Value - _position, 0, Int32.MaxValue); - var toRead = Math.Min(buffer.Length, (int)(_source.ChunkSize.Value - chunkOffset)); + var toRead = Math.Min(buffer.Length, chunk.Length - (int)chunkOffset); toRead = Math.Min(toRead, (int)readToEnd); - if (isLastChunk) - { - var lastChunkExtraSize = _source.Size.Value % _source.ChunkSize.Value; - if (lastChunkExtraSize > 0) - { - toRead = Math.Min(toRead, (int)lastChunkExtraSize); - } - } chunk.Slice((int)chunkOffset, toRead) .Span - .CopyTo(buffer.Span.SliceFast(0, toRead)); + .CopyTo(buffer.Span); _position += (ulong)toRead; Debug.Assert(_position <= _source.Size.Value, "Read more than the size of the stream"); return toRead; @@ -117,8 +100,9 @@ private async ValueTask> GetChunkAsync(ulong index, CancellationTok return memory!.Memory; } - var memoryOwner = _pool.Rent((int)_source.ChunkSize.Value); - await _source.ReadChunkAsync(memoryOwner.Memory, index, token); + var chunkSize = _source.GetChunkSize(index); + var memoryOwner = _pool.Rent(chunkSize); + await _source.ReadChunkAsync(memoryOwner.Memory[..chunkSize], index, token); _cache.Add(index, memoryOwner); return memoryOwner.Memory; } @@ -130,12 +114,39 @@ private Memory GetChunk(ulong index) return memory!.Memory; } - var memoryOwner = _pool.Rent((int)_source.ChunkSize.Value); - _source.ReadChunk(memoryOwner.Memory.Span, index); + var chunkSize = _source.GetChunkSize(index); + var memoryOwner = _pool.Rent(chunkSize); + var chunkMemory = memoryOwner.Memory[..chunkSize]; + _source.ReadChunk(chunkMemory.Span, index); _cache.Add(index, memoryOwner); - return memoryOwner.Memory; + return chunkMemory; } + private ulong FindChunkIndex(ulong position) + { + ulong low = 0, high = _source.ChunkCount - 1; + while (low <= high) + { + var mid = (low + high) / 2; + var startOffset = _source.GetOffset(mid); + var nextOffset = mid + 1 < _source.ChunkCount ? _source.GetOffset(mid + 1) : _source.Size.Value; + + if (position >= startOffset && position < nextOffset) + { + return mid; + } + + if (position < startOffset) + { + high = mid - 1; + } + else + { + low = mid + 1; + } + } + throw new InvalidOperationException("Position out of range."); + } /// public override long Seek(long offset, SeekOrigin origin) @@ -165,7 +176,7 @@ public override long Seek(long offset, SeekOrigin origin) /// public override void SetLength(long value) { - throw new NotImplementedException(); + throw new NotSupportedException(); } /// @@ -177,7 +188,7 @@ public override void SetLength(long value) /// public override void Write(byte[] buffer, int offset, int count) { - throw new NotImplementedException(); + throw new NotSupportedException(); } /// diff --git a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs index a2c74b33a3..562ecf8e4f 100644 --- a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs @@ -14,30 +14,27 @@ public interface IChunkedStreamSource public Size Size { get; } /// - /// The size of the chunks. Every chunk must be the same size, except for the last chunk which can be - /// smaller. The final chunk must not be empty or larger than the chunk size. All chunks (except the last) - /// must be no smaller than 16KB. + /// The number of chunks in the source. /// - public Size ChunkSize { get; } + public ulong ChunkCount { get; } /// - /// The number of chunks in the source. + /// Gets the starting offset of the given chunk index. /// - public ulong ChunkCount { get; } + public ulong GetOffset(ulong chunkIndex); + + /// + /// Gets the size of a chunk given its index. + /// + public int GetChunkSize(ulong chunkIndex); /// - /// Reads a chunk from the source into the buffer. The buffer size will always be the same as `ChunkSize` + /// Reads a chunk from the source into the buffer. /// - /// - /// - /// - /// public Task ReadChunkAsync(Memory buffer, ulong chunkIndex, CancellationToken token = default); /// - /// Reads a chunk from the source into the buffer. The buffer size will always be the same as `ChunkSize` + /// Reads a chunk from the source into the buffer. /// - /// - /// public void ReadChunk(Span buffer, ulong chunkIndex); } diff --git a/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs b/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs index 71b3c45029..6ed832b2e1 100644 --- a/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs +++ b/src/Networking/NexusMods.Networking.Steam/DepotChunkProvider.cs @@ -32,7 +32,16 @@ public DepotChunkProvider(Session session, AppId appId, DepotId depotId, Manifes public Size ChunkSize => Size.MB; public ulong ChunkCount => (ulong)_fileData.Chunks.Length; - + public ulong GetOffset(ulong chunkIndex) + { + return _chunksSorted[chunkIndex].Offset; + } + + public int GetChunkSize(ulong chunkIndex) + { + return (int)_chunksSorted[chunkIndex].UncompressedSize.Value; + } + public async Task ReadChunkAsync(Memory buffer, ulong chunkIndex, CancellationToken token = default) { var chunk = _chunksSorted[chunkIndex]; diff --git a/src/NexusMods.DataModel/NxFileStore.cs b/src/NexusMods.DataModel/NxFileStore.cs index c636829e73..47fa4cc3bf 100644 --- a/src/NexusMods.DataModel/NxFileStore.cs +++ b/src/NexusMods.DataModel/NxFileStore.cs @@ -367,6 +367,16 @@ public ChunkedArchiveStream(FileEntry entry, ParsedHeader header, Stream stream) public Size Size => Size.From(_entry.DecompressedSize); public Size ChunkSize => Size.From((ulong)_header.Header.ChunkSizeBytes); public ulong ChunkCount => (ulong)_entry.GetChunkCount(_header.Header.ChunkSizeBytes); + + public ulong GetOffset(ulong chunkIndex) + { + return (ulong)_header.Header.ChunkSizeBytes * chunkIndex; + } + + public int GetChunkSize(ulong chunkIndex) + { + return _header.Header.ChunkSizeBytes; + } public async Task ReadChunkAsync(Memory buffer, ulong localIndex, CancellationToken token = default) { diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs index 6bb5c4d778..83a1f35bd7 100644 --- a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs @@ -1,5 +1,6 @@ using FluentAssertions; using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.Hashes; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.Values; using NexusMods.Hashing.xxHash3; @@ -22,10 +23,11 @@ public async Task CanGetProductInfo() var largestFile = manifest.Files.OrderByDescending(f => f.Size).First(); await using var stream = session.GetFileStream(SdvAppId, manifest, largestFile.Path); - var hash = await stream.xxHash3Async(); - stream.Length.Should().Be((long)largestFile.Size.Value); + var multiHash = new MultiHasher(); + var result = await multiHash.HashStream(stream, CancellationToken.None); - manifest.Should().NotBeNull(); + stream.Length.Should().Be((long)largestFile.Size.Value); + result.Sha1.Should().Be(largestFile.Hash); } } diff --git a/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs b/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs index b88bf8e0bc..2e3362ddbe 100644 --- a/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs +++ b/tests/NexusMods.DataModel.Tests/ChunkedReaders/ChunkedReaderTests.cs @@ -1,4 +1,5 @@ -using System.Text; +using System.Diagnostics; +using System.Text; using FluentAssertions; using NexusMods.Abstractions.IO.ChunkedStreams; using NexusMods.Hashing.xxHash3; @@ -131,29 +132,50 @@ class ChunkedMemoryStream : IChunkedStreamSource { private readonly MemoryStream _ms; private readonly int _chunkSize; + private (ulong Offset, byte[] Data)[] _chunks; public ChunkedMemoryStream(MemoryStream ms, int chunkSize) { _chunkSize = chunkSize; _ms = ms; + + List<(ulong Offset, byte[] Data)> chunks = []; + var offset = 0; + var done = false; + while (!done) + { + var size = Random.Shared.Next(128, 1024); + if (offset + size > ms.Length) + { + size = (int)(ms.Length - offset); + done = true; + } + _ms.Position = offset; + var buffer = new byte[size]; + _ms.ReadExactly(buffer); + chunks.Add(((ulong)offset, buffer)); + offset += size; + } + _chunks = chunks.ToArray(); } public Size Size => Size.FromLong(_ms.Length); - public Size ChunkSize => Size.FromLong(_chunkSize); - public ulong ChunkCount => (ulong)Math.Ceiling(_ms.Length / (double)_chunkSize); + public ulong ChunkCount => (ulong)_chunks.Length; + public ulong GetOffset(ulong chunkIndex) => _chunks[chunkIndex].Offset; + public int GetChunkSize(ulong chunkIndex) => _chunks[chunkIndex].Data.Length; + public async Task ReadChunkAsync(Memory buffer, ulong chunkIndex, CancellationToken token = default) { - var offset = chunkIndex * (ulong)_chunkSize; - _ms.Position = (long)offset; - await _ms.ReadAsync(buffer, token); + await Task.Yield(); + Debug.Assert(buffer.Length == _chunks[chunkIndex].Data.Length); + _chunks[chunkIndex].Data.CopyTo(buffer.Span); } public void ReadChunk(Span buffer, ulong chunkIndex) { - var offset = chunkIndex * (ulong)_chunkSize; - _ms.Position = (long)offset; - _ms.Read(buffer); + Debug.Assert(buffer.Length == _chunks[chunkIndex].Data.Length); + _chunks[chunkIndex].Data.CopyTo(buffer); } } } From c90d1213a4bc34e1941350c1adecb0b25ae54921 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 12 Dec 2024 06:29:30 -0700 Subject: [PATCH 06/22] Delete the old Store project --- NexusMods.App.sln | 10 - NexusMods.Stores.Steam/CDNPool.cs | 9 - .../NexusMods.Stores.Steam.csproj | 20 -- NexusMods.Stores.Steam/Program.cs | 12 -- NexusMods.Stores.Steam/README.md | 6 - NexusMods.Stores.Steam/Session.cs | 200 ------------------ .../ChunkedStreams/ChunkedStream.cs | 21 +- .../ChunkedStreams/IChunkedStreamSource.cs | 2 +- .../NexusMods.Networking.Steam/CLI/Verbs.cs | 6 +- 9 files changed, 12 insertions(+), 274 deletions(-) delete mode 100644 NexusMods.Stores.Steam/CDNPool.cs delete mode 100644 NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj delete mode 100644 NexusMods.Stores.Steam/Program.cs delete mode 100644 NexusMods.Stores.Steam/README.md delete mode 100644 NexusMods.Stores.Steam/Session.cs diff --git a/NexusMods.App.sln b/NexusMods.App.sln index 4cfb5f5062..bad01875df 100644 --- a/NexusMods.App.sln +++ b/NexusMods.App.sln @@ -278,10 +278,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.MountAndBla EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Telemetry.Tests", "tests\NexusMods.Telemetry.Tests\NexusMods.Telemetry.Tests.csproj", "{336387F7-3635-43FE-9C23-CBC0CE534989}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Stores", "Stores", "{D5C9FBEA-5BD0-4879-B67B-C728D04206E8}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Stores.Steam", "NexusMods.Stores.Steam\NexusMods.Stores.Steam.csproj", "{2987630F-40E8-4A5E-A702-0B7641F345DA}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.Steam", "src\Networking\NexusMods.Networking.Steam\NexusMods.Networking.Steam.csproj", "{4A501BBB-389C-460C-B0C3-6F2F968773B1}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Steam", "Abstractions\NexusMods.Abstractions.Steam\NexusMods.Abstractions.Steam.csproj", "{24457AAA-8954-4BD6-8EB5-168EAC6EFB1B}" @@ -740,10 +736,6 @@ Global {336387F7-3635-43FE-9C23-CBC0CE534989}.Debug|Any CPU.Build.0 = Debug|Any CPU {336387F7-3635-43FE-9C23-CBC0CE534989}.Release|Any CPU.ActiveCfg = Release|Any CPU {336387F7-3635-43FE-9C23-CBC0CE534989}.Release|Any CPU.Build.0 = Release|Any CPU - {2987630F-40E8-4A5E-A702-0B7641F345DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2987630F-40E8-4A5E-A702-0B7641F345DA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2987630F-40E8-4A5E-A702-0B7641F345DA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2987630F-40E8-4A5E-A702-0B7641F345DA}.Release|Any CPU.Build.0 = Release|Any CPU {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Debug|Any CPU.Build.0 = Debug|Any CPU {4A501BBB-389C-460C-B0C3-6F2F968773B1}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -889,8 +881,6 @@ Global {A5A2932D-B3EF-480B-BEBC-793F6FC90EDE} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} {8D7E82BB-2F8D-455A-AF12-C486D9EC3B77} = {70D38D24-79AE-4600-8E83-17F3C11BA81F} {336387F7-3635-43FE-9C23-CBC0CE534989} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B} - {D5C9FBEA-5BD0-4879-B67B-C728D04206E8} = {E7BAE287-D505-4D6D-A090-665A64309B2D} - {2987630F-40E8-4A5E-A702-0B7641F345DA} = {D5C9FBEA-5BD0-4879-B67B-C728D04206E8} {4A501BBB-389C-460C-B0C3-6F2F968773B1} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C} {24457AAA-8954-4BD6-8EB5-168EAC6EFB1B} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C} {17023DB9-8E31-4397-B3E1-141149987865} = {897C4198-884F-448A-B0B0-C2A6D971EAE0} diff --git a/NexusMods.Stores.Steam/CDNPool.cs b/NexusMods.Stores.Steam/CDNPool.cs deleted file mode 100644 index 0975d1a55a..0000000000 --- a/NexusMods.Stores.Steam/CDNPool.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace NexusMods.Stores.Steam; - -public class CDNPool -{ - public CDNPool() - { - } - -} diff --git a/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj b/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj deleted file mode 100644 index 194da5f628..0000000000 --- a/NexusMods.Stores.Steam/NexusMods.Stores.Steam.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - Exe - net9.0 - enable - enable - - - - - - - - - - - - - diff --git a/NexusMods.Stores.Steam/Program.cs b/NexusMods.Stores.Steam/Program.cs deleted file mode 100644 index 555227af4c..0000000000 --- a/NexusMods.Stores.Steam/Program.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace NexusMods.Stores.Steam; - -public class Program -{ - public static async Task Main(string[] argv) - { - var client = new Session(); - await client.ConnectAsync(); - - return 0; - } -} diff --git a/NexusMods.Stores.Steam/README.md b/NexusMods.Stores.Steam/README.md deleted file mode 100644 index f10b2d774d..0000000000 --- a/NexusMods.Stores.Steam/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## NexusMods.Stores.Steam - -A minimalist API wrapper for accessing steam files via a Steam Login. For now this is used only -to collect and process hash data about games and their files. If this code ends up working -well enough we may integrate it with the larger app, but for now this is only for supporting -utilities. diff --git a/NexusMods.Stores.Steam/Session.cs b/NexusMods.Stores.Steam/Session.cs deleted file mode 100644 index 125bda0521..0000000000 --- a/NexusMods.Stores.Steam/Session.cs +++ /dev/null @@ -1,200 +0,0 @@ -using System.Collections.Concurrent; -using QRCoder; -using SteamKit2; -using SteamKit2.Authentication; -using SteamKit2.CDN; - -namespace NexusMods.Stores.Steam; - -public class Session -{ - private readonly SteamConfiguration _clientConfiguration; - private readonly SteamClient _steamClient; - private readonly CallbackManager _callbacks; - private readonly SteamUser _steamUser; - private readonly SteamApps _steamApps; - - private ConcurrentBag _licenses = []; - - private bool _running = false; - private readonly SteamContent _steamContent; - private readonly Client _cdnClient; - - public Session() - { - _steamClient = new SteamClient(); - _steamUser = _steamClient.GetHandler()!; - _steamApps = _steamClient.GetHandler()!; - _steamContent = _steamClient.GetHandler()!; - _cdnClient = new Client(_steamClient); - - _callbacks = new CallbackManager(_steamClient); - _callbacks.Subscribe(o => Task.Run(() => ConnectedCallback(o))); - _callbacks.Subscribe(DisconnectedCallback); - _callbacks.Subscribe(LoggedOnCallback); - _callbacks.Subscribe(LicenseListCallback); - _callbacks.Subscribe(o => Task.Run(() => PICSProductInfoCallback(o))); - } - - private Task PICSProductInfoCallback(SteamApps.PICSProductInfoCallback picsProductInfoCallback) - { - var result = new Dictionary(); - foreach (var kv in picsProductInfoCallback.Apps.First().Value.KeyValues.Children) - { - ToJson(result, kv); - } - throw new NotImplementedException(); - } - - private void ToJson(Dictionary result, KeyValue kv) - { - if (kv.Value != null) - result[kv.Name!] = kv.Value; - else - { - var subDict = new Dictionary(); - foreach (var subKv in kv.Children) - { - ToJson(subDict, subKv); - } - result[kv.Name!] = subDict; - } - } - - - public async Task ConnectAsync() - { - _running = true; - _steamClient.Connect(); - while (_running) - { - await _callbacks.RunWaitCallbackAsync(CancellationToken.None); - } - } - - - private void LicenseListCallback(SteamApps.LicenseListCallback obj) - { - foreach (var license in obj.LicenseList) - { - _licenses.Add(license); - } - } - - private void LoggedOnCallback(SteamUser.LoggedOnCallback callback) - { - if (callback.Result != EResult.OK) - { - Console.WriteLine("Unable to logon to Steam: {0} / {1}", callback.Result, callback.ExtendedResult); - _running = false; - return; - } - - Console.WriteLine("Successfully logged on!"); - - Console.WriteLine("Requesting license list..."); - //_steamApps.PICSGetProductInfo(new SteamApps.PICSRequest(413150), null); - - Task.Run(async () => await GetManifestInfo()); - } - - private async Task GetManifestInfo() - { - var appId = (uint)413150; - var depotId = (uint)413151; - var manifestId = 1364246008775303529UL; - var requestCode = await GetDepotManifestRequestCodeAsync(depotId, appId, manifestId, "public"); - var depotKey = await GetDepotKey(depotId, appId); - var servers = await _steamContent.GetServersForSteamPipe(); - var usable = servers - .Where(s => s.Type == "CDN") - .ToArray(); - Random.Shared.Shuffle(usable); - var server = usable.First(); - var cdnAuthToken = await RequestCDNAuthTokenAsync(appId, depotId, server); - - Console.WriteLine("Got {0} servers", servers.Count); - var manifest = await _cdnClient.DownloadManifestAsync(depotId, manifestId, requestCode, server, depotKey, cdnAuthToken: cdnAuthToken); - - Console.WriteLine("Got manifest with {0} files", manifest.Files.Count); - - } - - private async Task RequestCDNAuthTokenAsync(uint appId, uint depotId, Server server) - { - var cdnAuth = await _steamContent.GetCDNAuthToken(appId, depotId, server.Host!); - - if (cdnAuth.Result != EResult.OK) - Console.WriteLine("Failed to get CDN auth token for depot {0}", depotId); - else - Console.WriteLine("Got CDN auth token for depot {0}", depotId); - - return cdnAuth.Token; - } - - private async Task GetDepotKey(uint depotId, uint appId) - { - var key = await _steamApps.GetDepotDecryptionKey(depotId, appId); - if (key.Result != EResult.OK) - Console.WriteLine("Failed to get depot key for depot {0}", depotId); - else - Console.WriteLine("Got depot key for depot {0}", depotId); - - return key!.DepotKey; - } - - private async Task GetDepotManifestRequestCodeAsync(uint depotId, uint appId, ulong manifestId, string branch) - { - var requestCode = await _steamContent.GetManifestRequestCode(depotId, appId, manifestId, branch); - if (requestCode == 0) - throw new Exception("Unable to get request code for depot " + depotId + " manifest " + manifestId); - return requestCode; - } - - private void DisconnectedCallback(SteamClient.DisconnectedCallback obj) - { - throw new NotImplementedException(); - } - - private async Task ConnectedCallback(SteamClient.ConnectedCallback callback) - { - var authSession = await _steamClient.Authentication.BeginAuthSessionViaQRAsync(new AuthSessionDetails()); - - authSession.ChallengeURLChanged = () => - { - Console.WriteLine(); - Console.WriteLine("Steam has generated a new QR code for you to scan"); - - DrawQRCode(authSession); - }; - - DrawQRCode(authSession); - var pollResponse = await authSession.PollingWaitForResultAsync(); - - Console.WriteLine("Logging in as " + pollResponse.AccountName + "..."); - _steamUser.LogOn(new SteamUser.LogOnDetails - { - Username = pollResponse.AccountName, - AccessToken = pollResponse.RefreshToken, - } - ); - - } - - private void DrawQRCode(QrAuthSession authSession) - { - Console.WriteLine( $"Challenge URL: {authSession.ChallengeURL}" ); - Console.WriteLine(); - - // Encode the link as a QR code - using var qrGenerator = new QRCodeGenerator(); - var qrCodeData = qrGenerator.CreateQrCode( authSession.ChallengeURL, QRCodeGenerator.ECCLevel.L ); - using var qrCode = new AsciiQRCode( qrCodeData ); - var qrCodeAsAsciiArt = qrCode.GetGraphic( 1, drawQuietZones: false ); - - Console.WriteLine( "Use the Steam Mobile App to sign in via QR code:" ); - Console.WriteLine( qrCodeAsAsciiArt ); - } - - public SteamUser.LogOnDetails LogOnDetails { get; set; } -} diff --git a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs index ed475d5e5f..f9e47360c4 100644 --- a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/ChunkedStream.cs @@ -1,6 +1,5 @@ using System.Buffers; using System.Diagnostics; -using Reloaded.Memory.Extensions; namespace NexusMods.Abstractions.IO.ChunkedStreams; @@ -18,8 +17,6 @@ public class ChunkedStream : Stream where T : IChunkedStreamSource /// /// Main constructor, creates a new Chunked stream from the given source, and with a LRU cache of the given size /// - /// - /// public ChunkedStream(T source, int capacity = 16) { _position = 0; @@ -46,7 +43,7 @@ public override int Read(Span buffer) var chunkOffset = _position - _source.GetOffset(chunkIdx); var chunkSize = _source.GetChunkSize(chunkIdx); var chunk = GetChunk(chunkIdx)[..chunkSize]; - var readToEnd = Math.Clamp(_source.Size.Value - _position, 0, Int32.MaxValue); + var readToEnd = Math.Clamp(_source.Size.Value - _position, 0, int.MaxValue); var toRead = Math.Min(buffer.Length, chunk.Length - (int)chunkOffset); toRead = Math.Min(toRead, (int)readToEnd); @@ -122,6 +119,9 @@ private Memory GetChunk(ulong index) return chunkMemory; } + /// + /// Performs a binary search of the chunks to find the chunk index that contains the given position. + /// private ulong FindChunkIndex(ulong position) { ulong low = 0, high = _source.ChunkCount - 1; @@ -129,21 +129,20 @@ private ulong FindChunkIndex(ulong position) { var mid = (low + high) / 2; var startOffset = _source.GetOffset(mid); - var nextOffset = mid + 1 < _source.ChunkCount ? _source.GetOffset(mid + 1) : _source.Size.Value; + + ulong nextOffset; + if (mid + 1 < _source.ChunkCount) + nextOffset = _source.GetOffset(mid + 1); + else + nextOffset = _source.Size.Value; if (position >= startOffset && position < nextOffset) - { return mid; - } if (position < startOffset) - { high = mid - 1; - } else - { low = mid + 1; - } } throw new InvalidOperationException("Position out of range."); } diff --git a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs index 562ecf8e4f..c42680445d 100644 --- a/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs +++ b/src/Abstractions/NexusMods.Abstractions.IO/ChunkedStreams/IChunkedStreamSource.cs @@ -19,7 +19,7 @@ public interface IChunkedStreamSource public ulong ChunkCount { get; } /// - /// Gets the starting offset of the given chunk index. + /// Gets the starting offset (relative to the start of the file) of the given chunk index. /// public ulong GetOffset(ulong chunkIndex); diff --git a/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs b/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs index f8881f7f04..15ae6a4589 100644 --- a/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs +++ b/src/Networking/NexusMods.Networking.Steam/CLI/Verbs.cs @@ -47,15 +47,11 @@ private static async Task IndexSteamApp( await JsonSerializer.SerializeAsync(outputStream, productInfo, indentedOptions, token); } - //await renderer.RenderAsync(Renderable.TextLine("Product info written to: " + productFile)); + // For each depot and each manifest, download the manifest and index the files foreach (var depot in productInfo.Depots) { - //await renderer.RenderAsync(Renderable.TextLine("Depot: " + depot.DepotId)); foreach (var (branch, manifestInfo) in depot.Manifests) { - //await renderer.RenderAsync(Renderable.TextLine("Branch: " + branch)); - //await renderer.RenderAsync(Renderable.TextLine("Manifest: " + manifestInfo.ManifestId)); - var manifest = await steamSession.GetManifestContents(appId, depot.DepotId, manifestInfo.ManifestId, branch, token); var manifestPath = output / "stores" / "steam" / "manifests" / (manifest.ManifestId + ".json").ToRelativePath(); From eb32afded7e41408f84c310e8cbd0c09377465b6 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 12 Dec 2024 09:58:06 -0700 Subject: [PATCH 07/22] Disable the test that has to be run with human interaction --- .../NexusMods.Networking.Steam.Tests/BasicApiTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs index 83a1f35bd7..243a561f56 100644 --- a/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs +++ b/tests/Networking/NexusMods.Networking.Steam.Tests/BasicApiTests.cs @@ -3,7 +3,6 @@ using NexusMods.Abstractions.Hashes; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.Values; -using NexusMods.Hashing.xxHash3; using Xunit; namespace NexusMods.Networking.Steam.Tests; @@ -12,7 +11,7 @@ public class BasicApiTests(ILogger logger, ISteamSession session) { private static readonly AppId SdvAppId = AppId.From(413150); - [Fact] + [Fact(Skip = "Requires Human Interaction")] public async Task CanGetProductInfo() { var info = await session.GetProductInfoAsync(SdvAppId); From ed8f39fff05bd2f94e40fe2f80f036b88b62d839 Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 12 Dec 2024 09:59:44 -0700 Subject: [PATCH 08/22] Few more comments and cleanup --- .../NexusMods.Networking.Steam/Session.cs | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs index 5093b5440b..28bcdc1642 100644 --- a/src/Networking/NexusMods.Networking.Steam/Session.cs +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using System.Diagnostics; using Microsoft.Extensions.Logging; using NexusMods.Abstractions.IO.ChunkedStreams; using NexusMods.Abstractions.Steam; @@ -26,8 +25,7 @@ public class Session : ISteamSession private readonly ILogger _logger; private readonly IAuthInterventionHandler _handler; - private readonly SteamConfiguration _steamConfiguration; - + /// /// Base steam component, this is used for communicating with the Steam network. /// @@ -60,25 +58,26 @@ public class Session : ISteamSession private ConcurrentDictionary<(AppId, DepotId), byte[]> _depotKeys = new(); private ConcurrentDictionary<(AppId, DepotId, ManifestId, string Branch), ulong> _manifestRequestCodes = new(); - public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage, HttpClient httpClient) + public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage) { - _logger = logger; _handler = handler; _authStorage = storage; - _steamConfiguration = SteamConfiguration.Create(configurator => + var steamConfiguration = SteamConfiguration.Create(configurator => { + // The client will dispose of these on its own configurator.WithHttpClientFactory(() => new HttpClient()); }); - _steamClient = new SteamClient(_steamConfiguration); + _steamClient = new SteamClient(steamConfiguration); _steamUser = _steamClient.GetHandler()!; _steamApps = _steamClient.GetHandler()!; _steamContent = _steamClient.GetHandler()!; _cdnClient = new Client(_steamClient); _cdnPool = new CDNPool(this); - + // Some parts of this interface use callbacks instead of more natural async methods. So we need to register + // those callbacks here. _callbacks = new CallbackManager(_steamClient); _callbacks.Subscribe(WrapAsync(ConnectedCallback)); _callbacks.Subscribe(WrapAsync(DisconnectedCallback)); @@ -105,12 +104,7 @@ private Task LicenseListCallback(SteamApps.LicenseListCallback arg) { return Task.CompletedTask; } - - private async Task PICSProductInfoCallback(SteamApps.PICSProductInfoCallback callback) - { - _logger.LogInformation("Received PICSProductInfoCallback"); - } - + private async Task LoggedOnCallback(SteamUser.LoggedOnCallback callback) { _isLoggedOn = true; @@ -170,8 +164,7 @@ private Action WrapAsync(Func action) { return arg => Task.Run(async () => await action(arg)); } - - + public async Task GetProductInfoAsync(AppId appId, CancellationToken cancellationToken = default) { await ConnectedAsync(cancellationToken); From d64c215a8b9944c909c0db6f7c18c15c794db1d4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:26:06 -0700 Subject: [PATCH 09/22] Update Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs Co-authored-by: erri120 --- Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs index 8d6d6eae25..b74e4b1e1e 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs @@ -15,10 +15,10 @@ public class ProductInfo /// /// The app id of the product. /// - public AppId AppId { get; init; } + public required AppId AppId { get; init; } /// /// The depots of the product. /// - public Depot[] Depots { get; init; } + public required Depot[] Depots { get; init; } } From f734e431c0e8f652311a4f65f6125127edc20e52 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:27:18 -0700 Subject: [PATCH 10/22] Update src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs Co-authored-by: erri120 --- .../NexusMods.Abstractions.Hashes/Crc32.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs index 3aa1a6ae84..6457537cc2 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs @@ -23,17 +23,27 @@ internal class Crc32JsonConverter : JsonConverter { public override Crc32 Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - Span chars = stackalloc byte[4]; - Convert.FromHexString(reader.GetString()!, chars, out _, out _); - return Crc32.From(MemoryMarshal.Read(chars)); + var input = reader.GetString(); + if (input is null) throw new JsonException(); + + Span bytes = stackalloc byte[sizeof(uint)]; + + var status = Convert.FromHexString(input, bytes, out _, out _); + Debug.Assert(status == OperationStatus.Done); + + var value = MemoryMarshal.Read(bytes); + return Crc32.From(value); } public override void Write(Utf8JsonWriter writer, Crc32 value, JsonSerializerOptions options) { - Span span = stackalloc char[8]; - Span bytes = stackalloc byte[4]; + Span span = stackalloc char[sizeof(uint) * 2]; + Span bytes = stackalloc byte[sizeof(uint)]; MemoryMarshal.Write(bytes, value.Value); - Convert.TryToHexString(bytes, span, out _); + + var success = Convert.TryToHexString(bytes, span, out _); + Debug.Assert(success); + writer.WriteStringValue(span); } } From b58d34971835a525d3e8123957650c055b834791 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:27:25 -0700 Subject: [PATCH 11/22] Update src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj Co-authored-by: erri120 --- .../NexusMods.Networking.Steam.csproj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj index 0122d23079..e602918860 100644 --- a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj +++ b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -1,10 +1,5 @@  - - net9.0 - enable - enable - From 05e5f85c4b6121bfff5214c246b979bed4953f88 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:27:44 -0700 Subject: [PATCH 12/22] Update src/Networking/NexusMods.Networking.Steam/Session.cs Co-authored-by: erri120 --- src/Networking/NexusMods.Networking.Steam/Session.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs index 28bcdc1642..3e7ca15575 100644 --- a/src/Networking/NexusMods.Networking.Steam/Session.cs +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -224,11 +224,11 @@ public async Task GetDepotKey(AppId appId, DepotId depotId) var key = await _steamApps.GetDepotDecryptionKey(depotId.Value, appId.Value); if (key.Result != EResult.OK) { - _logger.LogWarning("Failed to get depot key for depot {0}", depotId.Value); - throw new Exception("Failed to get depot key for depot " + depotId.Value); + _logger.LogWarning("Failed to get depot key for depot `{DepotId}`", depotId.Value); + throw new Exception($"Failed to get depot key for depot `{depotId.Value}`"); } - _logger.LogInformation("Got depot key for depot {0}", depotId.Value); + _logger.LogInformation("Got depot key for depot `{DepotId}`", depotId.Value); _depotKeys.TryAdd((appId, depotId), key.DepotKey); return key.DepotKey; From 0c20eb3795d4d911106bf8eff70a82902695a051 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:28:56 -0700 Subject: [PATCH 13/22] Update Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs Co-authored-by: erri120 --- Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs index ae4a0b983b..602c273c86 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs @@ -54,12 +54,12 @@ public class FileData /// /// The Sha1 hash of the file /// - public Sha1 Hash { get; init; } + public required Sha1 Hash { get; init; } /// /// The chunks of the file /// - public Chunk[] Chunks { get; init; } + public required Chunk[] Chunks { get; init; } } public class Chunk From fd26dc246b1a471c755dd795f800adf5220bf453 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:30:59 -0700 Subject: [PATCH 14/22] Update src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs Co-authored-by: erri120 --- .../ProductInfoParser.cs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs index e304c34d98..de2489347a 100644 --- a/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs +++ b/src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs @@ -34,32 +34,32 @@ public static ProductInfo Parse(SteamApps.PICSProductInfoCallback callback) return productInfo; } - public static bool TryParseDownloadableDepot(AppId appId, KeyValue depot, out Depot result) + private static bool TryParseDownloadableDepot(AppId appId, KeyValue depot, [NotNullWhen(true)] out Depot? result) { if (!uint.TryParse(depot.Name, out var parsedDepotId)) { - result = default(Depot)!; + result = null; return false; } - + var configSection = depot.Children.FirstOrDefault(c => c.Name == "config"); var osList = configSection?.Children.FirstOrDefault(c => c.Name == "oslist")?.Value ?? ""; var depotId = DepotId.From(parsedDepotId); - + var manifestsKey = depot.Children.FirstOrDefault(c => c.Name == "manifests"); if (manifestsKey == null) { - result = default(Depot)!; + result = null; return false; } Dictionary manifestInfos = new(); foreach (var branch in manifestsKey.Children) { - var manifestId = ManifestId.From(ulong.Parse(branch.Children.First(f => f.Name == "gid").Value!)); - var sizeOnDisk = Size.From(ulong.Parse(branch.Children.First(f => f.Name == "size").Value!)); - var downloadSize = Size.From(ulong.Parse(branch.Children.First(f => f.Name == "download").Value!)); + var manifestId = ManifestId.From(ulong.Parse(branch["gid"].Value!)); + var sizeOnDisk = Size.From(ulong.Parse(branch["size"].Value!)); + var downloadSize = Size.From(ulong.Parse(branch["download"].Value!)); var manifestInfo = new ManifestInfo { @@ -67,16 +67,17 @@ public static bool TryParseDownloadableDepot(AppId appId, KeyValue depot, out De Size = sizeOnDisk, DownloadSize = downloadSize, }; - manifestInfos.Add(branch.Name!, manifestInfo); + + manifestInfos[branch.Name!] = manifestInfo; } - result = new Depot { DepotId = depotId, OsList = osList.Split(',', ' '), Manifests = manifestInfos, }; + return true; } From 869a9597b38caf1c2df84d13d873ea9ffe74be9d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:33:20 -0700 Subject: [PATCH 15/22] Update src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs Co-authored-by: erri120 --- src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs index 290cf2a75a..4d6726c64f 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs @@ -47,7 +47,7 @@ public async Task HashStream(Stream stream, CancellationToken token) while (true) { token.ThrowIfCancellationRequested(); - var read = await stream.ReadAsync(_buffer, 0, _buffer.Length, token); + var read = await stream.ReadAsync(_buffer, token); if (read == 0) break; var span = _buffer.AsSpan(0, read); From 752486b2103d3160025609e60ec9bb6bd154626d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 16 Dec 2024 06:34:00 -0700 Subject: [PATCH 16/22] Update src/Networking/NexusMods.Networking.Steam/Session.cs Co-authored-by: erri120 --- src/Networking/NexusMods.Networking.Steam/Session.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/Session.cs b/src/Networking/NexusMods.Networking.Steam/Session.cs index 3e7ca15575..251ea94063 100644 --- a/src/Networking/NexusMods.Networking.Steam/Session.cs +++ b/src/Networking/NexusMods.Networking.Steam/Session.cs @@ -168,11 +168,14 @@ private Action WrapAsync(Func action) public async Task GetProductInfoAsync(AppId appId, CancellationToken cancellationToken = default) { await ConnectedAsync(cancellationToken); + var jobs = await _steamApps.PICSGetProductInfo(new SteamApps.PICSRequest(appId.Value), null); - if (jobs.Failed) - throw new Exception("Failed to get product info for app " + appId.Value); + if (jobs.Failed) throw new Exception($"Failed to get product info for app `{appId}`"); + + var results = jobs.Results; + if (results is null || results.Count == 0) throw new Exception($"Found no product info for app `{appId}`"); - return ProductInfoParser.Parse(jobs.Results![0]); + return ProductInfoParser.Parse(results[0]); } /// From 6815ce463635f8d75e62f7276d08570243e006d5 Mon Sep 17 00:00:00 2001 From: halgari Date: Mon, 16 Dec 2024 08:41:22 -0700 Subject: [PATCH 17/22] `sizeof(ulong)` vs `size(long)` --- src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs index 4d6726c64f..15c4304c9e 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs @@ -108,8 +108,8 @@ public static async Task MinimalHash(XxHash3 hasher, Stream stream, Cancellation await stream.ReadExactlyAsync(buffer, cancellationToken); hasher.Append(buffer.Span); - // Add the length of the file to the hash (as a ulong) - Span lengthBuffer = stackalloc byte[sizeof(long)]; + // Add the length of the file to the hash (as an ulong) + Span lengthBuffer = stackalloc byte[sizeof(ulong)]; MemoryMarshal.Write(lengthBuffer, (ulong)stream.Length); hasher.Append(lengthBuffer); } From f6e59ce20c816ca1847833de2c710b35b10a089b Mon Sep 17 00:00:00 2001 From: halgari Date: Mon, 16 Dec 2024 08:42:39 -0700 Subject: [PATCH 18/22] Update doc strings to better explain the value sizes --- .../NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs index 8c48741cb5..e2aeb3c6d3 100644 --- a/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs +++ b/Abstractions/NexusMods.Abstractions.Steam/DTOs/ManifestInfo.cs @@ -15,12 +15,12 @@ public class ManifestInfo public required ManifestId ManifestId { get; init; } /// - /// The size of the downloaded files + /// The size of the downloaded files, decompressed /// public required Size Size { get; init; } /// - /// The size of the files on the CDN + /// The size of the files, compressed /// public required Size DownloadSize { get; init; } } From 782c74fcc06dbae303723073dcb973f40157b126 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Dec 2024 11:30:16 -0700 Subject: [PATCH 19/22] Update src/Networking/NexusMods.Networking.Steam/ManifestParser.cs Co-authored-by: erri120 --- .../ManifestParser.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs b/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs index 153bbc2219..e3ae042e7d 100644 --- a/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs +++ b/src/Networking/NexusMods.Networking.Steam/ManifestParser.cs @@ -10,6 +10,21 @@ public static class ManifestParser { public static Manifest Parse(DepotManifest manifest) { + var files = manifest.Files is null ? [] : manifest.Files.Select(file => new Manifest.FileData + { + Path = file.FileName, + Size = Size.From(file.TotalSize), + Hash = Sha1.From(file.FileHash), + Chunks = file.Chunks.Select(chunk => new Manifest.Chunk + { + ChunkId = Sha1.From(chunk.ChunkID), + Offset = chunk.Offset, + CompressedSize = Size.From(chunk.CompressedLength), + UncompressedSize = Size.From(chunk.UncompressedLength), + Checksum = Crc32.From(chunk.Checksum), + }).ToArray(), + }).ToArray(); + return new Manifest { ManifestId = ManifestId.From(manifest.ManifestGID), @@ -17,22 +32,7 @@ public static Manifest Parse(DepotManifest manifest) TotalUncompressedSize = Size.From(manifest.TotalUncompressedSize), CreationTime = manifest.CreationTime, DepotId = DepotId.From(manifest.DepotID), - Files = manifest.Files!.Select(file => new Manifest.FileData - { - Path = file.FileName, - Size = Size.From(file.TotalSize), - Hash = Sha1.From(file.FileHash), - Chunks = file.Chunks.Select(chunk => new Manifest.Chunk - { - ChunkId = Sha1.From(chunk.ChunkID), - Offset = chunk.Offset, - CompressedSize = Size.From(chunk.CompressedLength), - UncompressedSize = Size.From(chunk.UncompressedLength), - Checksum = Crc32.From(chunk.Checksum), - } - ).ToArray(), - } - ).ToArray(), + Files = files, }; } From a5dd382f7323db67e6e0e68e6a9c00a20ce8860c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Dec 2024 11:32:55 -0700 Subject: [PATCH 20/22] Update src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs Co-authored-by: erri120 --- .../HashJsonConverter.cs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs index 612586befb..7041253ade 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs @@ -10,19 +10,31 @@ namespace NexusMods.Abstractions.Hashes; /// public class HashJsonConverter : JsonConverter { + /// public override Hash Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - Span chars = stackalloc byte[8]; - Convert.FromHexString(reader.GetString()!, chars, out _, out _); - return Hash.FromULong(MemoryMarshal.Read(chars)); + var input = reader.GetString(); + if (input is null) throw new JsonException(); + + Span bytes = stackalloc byte[sizeof(ulong)]; + + var status = Convert.FromHexString(input, bytes, out _, out _); + Debug.Assert(status == OperationStatus.Done); + + var value = MemoryMarshal.Read(bytes); + return Hash.FromULong(value); } + /// public override void Write(Utf8JsonWriter writer, Hash value, JsonSerializerOptions options) { - Span span = stackalloc char[16]; - Span bytes = stackalloc byte[8]; + Span span = stackalloc char[sizeof(ulong) * 2]; + Span bytes = stackalloc byte[sizeof(ulong)]; MemoryMarshal.Write(bytes, value.Value); - Convert.TryToHexString(bytes, span, out _); + + var success = Convert.TryToHexString(bytes, span, out _); + Debug.Assert(success); + writer.WriteStringValue(span); } } From 6cca217a42977e508252440feb1d3add8c4c185c Mon Sep 17 00:00:00 2001 From: halgari Date: Tue, 17 Dec 2024 11:33:48 -0700 Subject: [PATCH 21/22] Fix the .Steam project --- .../NexusMods.Networking.Steam.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj index e602918860..28985a4233 100644 --- a/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj +++ b/src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -1,5 +1,5 @@  - + From f2b3e6b6e12a373ab9eb30eaf94c07745bd54bc2 Mon Sep 17 00:00:00 2001 From: halgari Date: Tue, 17 Dec 2024 11:38:52 -0700 Subject: [PATCH 22/22] fix compile errors introduced during code review --- src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs | 2 ++ .../NexusMods.Abstractions.Hashes/HashJsonConverter.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs index 6457537cc2..44073e4770 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs @@ -1,3 +1,5 @@ +using System.Buffers; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Text.Json; using System.Text.Json.Serialization; diff --git a/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs index 7041253ade..5f02bcd06f 100644 --- a/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs +++ b/src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs @@ -1,3 +1,5 @@ +using System.Buffers; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Text.Json; using System.Text.Json.Serialization;