From 60baa4199f0b7333d4848f127cfcc52a7d4028a6 Mon Sep 17 00:00:00 2001 From: maddie480 <52103563+maddie480@users.noreply.github.com> Date: Sun, 12 Nov 2023 12:11:49 +0100 Subject: [PATCH] Add mod alias support --- README.md | 4 +- pom.xml | 2 +- .../updatechecker/ConnectionUtils.java | 2 +- .../updatechecker/DatabaseUpdater.java | 3 +- .../everest/updatechecker/EventListener.java | 4 + .../everest/updatechecker/ModAliasLister.java | 122 ++++++++++++++++++ 6 files changed, 132 insertions(+), 5 deletions(-) create mode 100755 src/main/java/ovh/maddie480/everest/updatechecker/ModAliasLister.java diff --git a/README.md b/README.md index a21ce0e..9245883 100755 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ Get Maven, then run the following command at the project root: mvn clean package ``` -This will build the project to `target/update-checker-0.4.12.jar`. +This will build the project to `target/update-checker-0.5.0.jar`. ### Running the project @@ -213,7 +213,7 @@ First, follow these steps to set it up: Then, to run the project, browse to where the JAR is in a terminal / command prompt, then run ``` -java -jar update-checker-0.4.12.jar [minutes] +java -jar update-checker-0.5.0.jar [minutes] ``` [minutes] is the wait delay in minutes between two GameBanana checks (defaults to 30). Be aware that the program makes ~13 API calls per check, and that the GameBanana API has a cap at 250 requests/hour. diff --git a/pom.xml b/pom.xml index 84a8661..8b25d7c7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ ovh.maddie480.everest.updatechecker update-checker - 0.4.12 + 0.5.0 21 diff --git a/src/main/java/ovh/maddie480/everest/updatechecker/ConnectionUtils.java b/src/main/java/ovh/maddie480/everest/updatechecker/ConnectionUtils.java index 3993ecb..b7e6cfa 100755 --- a/src/main/java/ovh/maddie480/everest/updatechecker/ConnectionUtils.java +++ b/src/main/java/ovh/maddie480/everest/updatechecker/ConnectionUtils.java @@ -32,7 +32,7 @@ public static HttpURLConnection openConnectionWithTimeout(String url) throws IOE throw new IOException(e); } - con.setRequestProperty("User-Agent", "Everest-Update-Checker/0.4.12 (+https://github.com/maddie480/EverestUpdateCheckerServer)"); + con.setRequestProperty("User-Agent", "Everest-Update-Checker/0.5.0 (+https://github.com/maddie480/EverestUpdateCheckerServer)"); con.setRequestProperty("Accept-Encoding", "gzip"); con.setConnectTimeout(10000); diff --git a/src/main/java/ovh/maddie480/everest/updatechecker/DatabaseUpdater.java b/src/main/java/ovh/maddie480/everest/updatechecker/DatabaseUpdater.java index 6a53172..17c5e11 100755 --- a/src/main/java/ovh/maddie480/everest/updatechecker/DatabaseUpdater.java +++ b/src/main/java/ovh/maddie480/everest/updatechecker/DatabaseUpdater.java @@ -165,8 +165,9 @@ private void saveDatabaseToYaml() throws IOException { new BananaMirrorRichPresenceIcons().update(modSearchDatabaseBuilder.getNsfwMods()); } - // update the dependency graph with new entries. + // update the dependency graph and mod aliases with new entries. DependencyGraphBuilder.updateDependencyGraph(); + ModAliasLister.updateModAliasList(); } /** diff --git a/src/main/java/ovh/maddie480/everest/updatechecker/EventListener.java b/src/main/java/ovh/maddie480/everest/updatechecker/EventListener.java index 4058e03..a30c5e5 100755 --- a/src/main/java/ovh/maddie480/everest/updatechecker/EventListener.java +++ b/src/main/java/ovh/maddie480/everest/updatechecker/EventListener.java @@ -50,6 +50,8 @@ static void handle(Consumer functionCall) { public abstract void scannedModDependencies(String modId, int dependencyCount, int optionalDependencyCount); + public abstract void scannedModAliases(String modId, List alsoKnownAsModIds); + public abstract void modUpdatedIncrementally(String gameBananaType, int gameBananaId, String modName); @@ -81,6 +83,8 @@ static void handle(Consumer functionCall) { public abstract void dependencyTreeScanException(String modId, Exception e); + public abstract void modAliasListScanException(String modId, Exception e); + public abstract void zipFileWalkthroughError(String gameBananaType, int gameBananaId, String fileUrl, Exception e); public abstract void ahornPluginScanError(String fileUrl, Exception e); diff --git a/src/main/java/ovh/maddie480/everest/updatechecker/ModAliasLister.java b/src/main/java/ovh/maddie480/everest/updatechecker/ModAliasLister.java new file mode 100755 index 0000000..c241eaa --- /dev/null +++ b/src/main/java/ovh/maddie480/everest/updatechecker/ModAliasLister.java @@ -0,0 +1,122 @@ +package ovh.maddie480.everest.updatechecker; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static ovh.maddie480.everest.updatechecker.DatabaseUpdater.checkZipSignature; + +public class ModAliasLister { + private static final Logger log = LoggerFactory.getLogger(ModAliasLister.class); + + static void updateModAliasList() throws IOException { + Map> oldAliasList; + try (InputStream is = Files.newInputStream(Paths.get("uploads/modaliases.yaml"))) { + oldAliasList = YamlUtil.load(is); + } + + Map> everestUpdate; + try (InputStream is = Files.newInputStream(Paths.get("uploads/everestupdate.yaml"))) { + everestUpdate = YamlUtil.load(is); + } + + Map> newAliasList = new HashMap<>(); + + // go across every entry in everest_update.yaml. + for (Map.Entry> mod : everestUpdate.entrySet()) { + String name = mod.getKey(); + String url = (String) mod.getValue().get(Main.serverConfig.mainServerIsMirror ? "MirrorURL" : "URL"); + + if (oldAliasList.containsKey(name) && oldAliasList.get(name).get("URL").toString().equals(url)) { + // we already have that mod! + newAliasList.put(name, oldAliasList.get(name)); + } else { + // download file + ConnectionUtils.runWithRetry(() -> { + try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(Paths.get("mod-aliaslist.zip")))) { + IOUtils.copy(new BufferedInputStream(ConnectionUtils.openStreamWithTimeout(url)), os); + return null; // to fulfill this stupid method signature + } + }); + + // check that its size makes sense + long actualSize = new File("mod-aliaslist.zip").length(); + if (((int) mod.getValue().get("Size")) != actualSize) { + FileUtils.forceDelete(new File("mod-aliaslist.zip")); + throw new IOException("The announced file size (" + mod.getValue().get("Size") + ") does not match what we got (" + actualSize + ")" + + " for file " + url); + } + + // read its everest.yaml and collect the values of AlsoKnownAs + List aliasList = Collections.emptyList(); + + try (ZipFile zipFile = ZipFileWithAutoEncoding.open("mod-aliaslist.zip")) { + checkZipSignature(new File("mod-aliaslist.zip").toPath()); + + ZipEntry everestYaml = zipFile.getEntry("everest.yaml"); + if (everestYaml == null) { + everestYaml = zipFile.getEntry("everest.yml"); + } + + List> everestYamlContents; + try (InputStream is = zipFile.getInputStream(everestYaml)) { + everestYamlContents = YamlUtil.loadNoFloats(is); + } + + Map matchingYamlEntry = null; + + for (Map yamlEntry : everestYamlContents) { + if (name.equals(yamlEntry.get("Name"))) { + matchingYamlEntry = yamlEntry; + break; + } + } + + if (matchingYamlEntry == null) { + throw new IOException("Could not find matching entry that's supposed to be there since it's in everestupdate.yaml!"); + } + + // instanceof List isn't a thing, but we can check if all elements are Strings instead! + if (matchingYamlEntry.get("AlsoKnownAs") instanceof List aliases + && aliases.stream().allMatch(e -> e instanceof String)) { + + aliasList = (List) aliases; + } + + log.info("Found aliases for {}: {}.", name, aliasList); + final List aliases = aliasList; + EventListener.handle(listener -> listener.scannedModAliases(name, aliases)); + } catch (Exception e) { + // if a file cannot be read as a zip, no need to worry about it. + // we will just write an empty array. + log.warn("Could not read aliases from {}", name, e); + EventListener.handle(listener -> listener.modAliasListScanException(name, e)); + } + + // save the entry we just got. + Map aliasEntry = new HashMap<>(); + aliasEntry.put("URL", url); + aliasEntry.put("AlsoKnownAs", aliasList); + newAliasList.put(name, aliasEntry); + + FileUtils.forceDelete(new File("mod-aliaslist.zip")); + } + } + + // write it out! + try (OutputStream os = new FileOutputStream("uploads/modaliases.yaml")) { + YamlUtil.dump(newAliasList, os); + } + } +}