Skip to content

Commit

Permalink
Collections Install Spike (#1996)
Browse files Browse the repository at this point in the history
* Add GraphQL support

* Bit of cleanup

* Can get collections download links from Nexus Mods, as part of NexusApiClient.cs

* Can download collections and link them to the download metadata

* Verify the contents of the downloaded collection

* Can install a basic collection

* Hook collections into the CLI and the NXM IPC handler

* Fix the download progress button

* Add support for `hashes` in collections

* Fix a stupid bug in NXM handling

* Package the collections project correctly for Nuget

* Add some basic documentation about how we download and use collections

* Handle PR feedback

* Remove comment

* Move logic out of the switch statements in NxmIpcProtocolHandler.cs

* Dedupe some code in the NexusModsLibrary.cs file

* Fix build issue

* Store various choices and source types as enums

* Remove uses of `_context` don't store the context locally

* Handle PR feedback, don't save the context off

* Integrate the collections install code with the previously added collections groups

* Add collections to the left menu

* Fix a few bugs with how collection data is passed to the mod installer
  • Loading branch information
halgari authored Sep 11, 2024
1 parent df1f68a commit c0db8e8
Show file tree
Hide file tree
Showing 76 changed files with 4,955 additions and 91 deletions.
2 changes: 2 additions & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<PackageVersion Include="FlatSharp.Runtime" Version="7.6.0" />
<PackageVersion Include="LinqGen" Version="0.3.1" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.8.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
<PackageVersion Include="Nerdbank.Streams" Version="2.11.74" />
<PackageVersion Include="NexusMods.Paths" Version="0.10.0" />
Expand All @@ -31,6 +32,7 @@
<PackageVersion Include="R3Extensions.Avalonia" Version="1.2.8" />
<PackageVersion Include="ReactiveUI" Version="20.1.1" />
<PackageVersion Include="Spectre.Console.Testing" Version="0.49.1" />
<PackageVersion Include="StrawberryShake.Server" Version="13.9.12" />
<PackageVersion Include="System.Linq" Version="4.3.0" />
<PackageVersion Include="System.Text.Json" Version="8.0.4" />
<PackageVersion Include="TextMateSharp.Grammars" Version="1.0.59" />
Expand Down
20 changes: 20 additions & 0 deletions NexusMods.App.sln
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.GC",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.FileStore.Nx", "src\Abstractions\NexusMods.Abstractions.FileStore.Nx\NexusMods.Abstractions.FileStore.Nx.csproj", "{5CB12332-A2E9-4A6A-993E-718490C61A9B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Collections", "src\Abstractions\NexusMods.Abstractions.Collections\NexusMods.Abstractions.Collections.csproj", "{BF6EEEA3-9C9C-404E-9B2D-6926EF503384}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Collections", "src\NexusMods.Collections\NexusMods.Collections.csproj", "{A9FD538A-E101-4AEA-A98E-35DCED950AEE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Collections.Tests", "tests\NexusMods.Collections.Tests\NexusMods.Collections.Tests.csproj", "{8C817874-7A88-450E-B216-851A1B03684C}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Media", "src\Abstractions\NexusMods.Abstractions.Media\NexusMods.Abstractions.Media.csproj", "{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}"
EndProject
Global
Expand Down Expand Up @@ -640,6 +645,18 @@ Global
{5CB12332-A2E9-4A6A-993E-718490C61A9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5CB12332-A2E9-4A6A-993E-718490C61A9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5CB12332-A2E9-4A6A-993E-718490C61A9B}.Release|Any CPU.Build.0 = Release|Any CPU
{BF6EEEA3-9C9C-404E-9B2D-6926EF503384}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BF6EEEA3-9C9C-404E-9B2D-6926EF503384}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BF6EEEA3-9C9C-404E-9B2D-6926EF503384}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BF6EEEA3-9C9C-404E-9B2D-6926EF503384}.Release|Any CPU.Build.0 = Release|Any CPU
{A9FD538A-E101-4AEA-A98E-35DCED950AEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A9FD538A-E101-4AEA-A98E-35DCED950AEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A9FD538A-E101-4AEA-A98E-35DCED950AEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A9FD538A-E101-4AEA-A98E-35DCED950AEE}.Release|Any CPU.Build.0 = Release|Any CPU
{8C817874-7A88-450E-B216-851A1B03684C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8C817874-7A88-450E-B216-851A1B03684C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8C817874-7A88-450E-B216-851A1B03684C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8C817874-7A88-450E-B216-851A1B03684C}.Release|Any CPU.Build.0 = Release|Any CPU
{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -758,6 +775,9 @@ Global
{B917FBF8-46F8-4776-8BAE-A54BED756E7E} = {D49730E2-2EED-4E72-88CA-35462CC8D9A6}
{A1FCEB06-7599-44AD-90CA-2C88C32B959C} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
{5CB12332-A2E9-4A6A-993E-718490C61A9B} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
{BF6EEEA3-9C9C-404E-9B2D-6926EF503384} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
{A9FD538A-E101-4AEA-A98E-35DCED950AEE} = {E7BAE287-D505-4D6D-A090-665A64309B2D}
{8C817874-7A88-450E-B216-851A1B03684C} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B}
{5CB6D02C-07D0-4C0D-BF5C-4E2E958A0612} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
Expand Down
186 changes: 186 additions & 0 deletions docs/developers/misc/CollectionsOverview.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
## Collections Overview

### Introduction
Collections are, as the name suggests, mostly a collection (really a list) of mods. At their core they consist of a `.zip`
file that contains a `collection.json` file and various other files bundled with it. Most of the mods are stored as a Nexus
mod ID and fileId, and are therefore sourced from the normal download APIs.

### Collections Metadata
Most of the collections metadata is stored and read from the Nexus Mods v2 GraphQL Api. The three main types of ids to be
aware of are:

- `slug` - a unique identifier for the collection, a 7 character alphanumeric string
- `revisionNumber` - a number that increments every time the collection is updated. It's unique only to the collection
- `revisionId` - a unique id for the revision of the collection, globally unique. Most likely this is an id in the Nexus Mods database

Based on a slug, GraphQL can be used to query the collection metadata, including the revision number and revision id. Based on
a revision id, and a slug, the download links for the collection can be obtained. Those links are then used to download the
collection `.zip` file. Inside that file is a `collection.json` file that contains the metadata for the collection.

Thus, the process for downloading a collection, given a slug and revision number are as follows:

1. Query the Nexus Mods v2 GraphQL API for the collection metadata. Each revision of the response data will include a `download_links`
field.
2. Use the `download_links` field to get a list of download links for the collection (one for each mirror and the CDN, just like files)
3. Download the collection `.zip` file from one of the download links
4. Analyze and add the collection file to the Library, and tag it with the collection metadata (mainly the slug and revision number)
5. Recast the collection file from the library as a `LibraryArchive` model, and find a child with the name `collection.json`
6. Parse the `collection.json` file


### Collection JSON

Here is an example `collection.json` file for reference

```json
{
"info": {
"author": "Anonymous",
"authorUrl": "",
"name": "Halgari's Helper",
"description": "",
"installInstructions": "",
"domainName": "cyberpunk2077",
"gameVersions": [
"3.0.76.64179"
]
},
"mods": [
{
"name": "Appearance Menu Mod",
"version": "2.7",
"optional": false,
"domainName": "cyberpunk2077",
"source": {
"type": "nexus",
"modId": 790,
"fileId": 66386,
"md5": "0a6e3e603ef3bca799436f69510c79b7",
"fileSize": 140159937,
"logicalFilename": "Appearance Menu Mod",
"updatePolicy": "prefer",
"tag": "JqF6xzzWA"
},
"hashes": [
{
"path": "archive\\pc\\mod\\AMM_Dino_TattooFix.archive",
"md5": "add39f916aa4f469b51881fe6b50a9c6"
},
{
"path": "archive\\pc\\mod\\AMM_RitaWheeler_CombatEnabler.archive",
"md5": "e1a03cf9eeb34288cb2d013f61381f63"
}
],
"author": "MaximiliumM and CtrlAltDaz",
"details": {
"category": "Appearance",
"type": ""
},
"phase": 0
},
{
"name": "Cyber Engine Tweaks - CET 1.32.2",
"version": "1.32.2",
"optional": false,
"domainName": "cyberpunk2077",
"source": {
"type": "nexus",
"modId": 107,
"fileId": 73822,
"md5": "4b1dd024876fdddfef2a2383492e1c1c",
"fileSize": 34849878,
"logicalFilename": "CET 1.32.2",
"updatePolicy": "prefer",
"tag": "x_A_Q2gQ3e"
},
"author": "yamashi",
"details": {
"category": "Modders Resources",
"type": ""
},
"phase": 0
},
{
"name": "Load Begone (Intro Splash Load and Checkpoint Removal - FOMOD) - Load Begone - 2.2.1 (FOMOD)",
"version": "2.2.1",
"optional": false,
"domainName": "cyberpunk2077",
"source": {
"type": "nexus",
"modId": 8144,
"fileId": 59926,
"md5": "f86b6241862c140891771306282abbf9",
"fileSize": 3911564,
"logicalFilename": "Load Begone - 2.2.1 (FOMOD)",
"updatePolicy": "prefer",
"tag": "vACbpm9SFd"
},
"choices": {
"type": "fomod",
"options": [
{
"name": "Installation",
"groups": [
{
"name": "Features",
"choices": [
{
"name": "Skip Intro Logos",
"idx": 0
},
{
"name": "No Splash Video",
"idx": 1
},
{
"name": "Faster Checkpoints",
"idx": 2
}
]
}
]
}
]
},
"author": "CyanideX",
"details": {
"category": "User Interface",
"type": ""
},
"phase": 0
}
],
"modRules": [],
"loadOrder": [],
"tools": [],
"collectionConfig": {
"recommendNewProfile": false
}
}



```

This data format isn't too complicated, and it is mostly self-explanatory. The `info` field contains metadata about the collection,
such as the author, name, description, and game versions. The `mods` field contains a list of mods in the collection. Each `mod` may
be sourced from one of several locations based on the `source` field.

The way the mod is installed is influenced by the `hashes` field and the `choices` field. The `hashes` field indicates a list of files
from the mod that should be installed to a given path, indicated by the MD5 hash. This method is used when the user selects `Replicate` as the mod mode when
creating the collection. Installation with this method is fairly simple, the files in the mod are hashed and then copied based on the input hashes.

The `choices` field is used when the user selects `Fomod` as the mod mode when creating the collection. This is more complex, and is not yet implemented
in the app.


## Known Unknowns

Here are some unknows that need to be resolved before collections can be fully implemented:

- What are phases? And do they matter at all for our purposes?
- What other types of `source` fields are there?
- Are there any other fields besides `hashes` and `choices` that are relevant to the installation process?
- What other sorts of installer `choices` are there besides `fomod`?
- What is the `tag` field in the `source` object?

Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using System.Text.Json.Serialization;

namespace NexusMods.Abstractions.Collections.Json;

public class Choices
{
[JsonPropertyName("type")]
[JsonConverter(typeof(JsonStringEnumConverter))]
public required ChoicesType Type { get; init; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// ReSharper disable InconsistentNaming
namespace NexusMods.Abstractions.Collections.Json;

/// <summary>
/// The types of choices on a mod
/// </summary>
public enum ChoicesType
{
fomod
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
using System.Text.Json.Serialization;
using NexusMods.Abstractions.Games.DTO;

namespace NexusMods.Abstractions.Collections.Json;

public class CollectionInfo
{
[JsonPropertyName("author")]
public string Author { get; init; } = string.Empty;

[JsonPropertyName("authorUrl")]
public string AuthorUrl { get; init; } = string.Empty;

[JsonPropertyName("name")]
public string Name { get; init; } = string.Empty;

[JsonPropertyName("description")]
public string Description { get; init; } = string.Empty;

[JsonPropertyName("installInstructions")]
public string InstallInstructions { get; init; } = string.Empty;

[JsonPropertyName("domainName")]
public required GameDomain DomainName { get; init; }

[JsonPropertyName("gameVersions")]
public string[] GameVersions { get; init; } = [];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;

namespace NexusMods.Abstractions.Collections.Json;

public class CollectionRoot
{
[JsonPropertyName("info")]
public required CollectionInfo Info { get; init; }

[JsonPropertyName("mods")]
public Mod[] Mods { get; init; } = [];
}
31 changes: 31 additions & 0 deletions src/Abstractions/NexusMods.Abstractions.Collections/Json/Mod.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
using System.Text.Json.Serialization;
using NexusMods.Abstractions.Games.DTO;

namespace NexusMods.Abstractions.Collections.Json;

public class Mod
{
/// <summary>
/// The name of the mod
/// </summary>
[JsonPropertyName("name")]
public string Name { get; init; } = string.Empty;

[JsonPropertyName("version")]
public string Version { get; init; } = string.Empty;

[JsonPropertyName("optional")]
public bool Optional { get; init; }

[JsonPropertyName("domainName")]
public required GameDomain DomainName { get; init; }

[JsonPropertyName("source")]
public required ModSource Source { get; init; }

[JsonPropertyName("hashes")]
public PathAndHash[] Hashes { get; init; } = [];

[JsonPropertyName("choices")]
public Choices? Choices { get; init; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using System.Text.Json.Serialization;
using NexusMods.Abstractions.NexusWebApi.Types;
using NexusMods.Paths;

namespace NexusMods.Abstractions.Collections.Json;

public class ModSource
{
[JsonPropertyName("type")]
[JsonConverter(typeof(JsonStringEnumConverter))]
public required ModSourceType Type { get; init; }

[JsonPropertyName("modId")]
public ModId ModId { get; init; }

[JsonPropertyName("fileId")]
public FileId FileId { get; init; }

[JsonPropertyName("fileSize")]
public Size FileSize { get; init; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// ReSharper disable InconsistentNaming
namespace NexusMods.Abstractions.Collections.Json;

/// <summary>
/// The possible sources of a mod
/// </summary>
public enum ModSourceType
{
nexus,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Text.Json.Serialization;
using NexusMods.Abstractions.Collections.Types;
using NexusMods.Paths;
// ReSharper disable InconsistentNaming

namespace NexusMods.Abstractions.Collections.Json;

/// <summary>
/// Path and MD5 hash value as a pair.
/// </summary>
public class PathAndHash
{
/// <summary>
/// Relative path for the file's installation location.
/// </summary>
[JsonPropertyName("path")]
public required RelativePath Path { get; init; }

/// <summary>
/// The MD5 hash of the file to put in that location.
/// </summary>
[JsonPropertyName("md5")]
public required Md5HashValue MD5 { get; init; }
}
Loading

0 comments on commit c0db8e8

Please sign in to comment.