From 5c4663dace9c5f00af3ef0b03370f3752545d6bf Mon Sep 17 00:00:00 2001 From: Matyrobbrt Date: Sun, 8 Dec 2024 17:32:12 +0200 Subject: [PATCH] Initial work on caching scan result --- .../net/neoforged/fml/loading/FMLPaths.java | 5 + .../fml/loading/moddiscovery/ModFile.java | 41 ++- .../fml/loading/modscan/Scanner.java | 35 ++- .../neoforgespi/language/ModFileScanData.java | 284 +++++++++++++++++- .../net/neoforged/fml/test/TestModFile.java | 6 +- .../neoforgespi/language/ScanDataTest.java | 70 +++++ 6 files changed, 429 insertions(+), 12 deletions(-) diff --git a/loader/src/main/java/net/neoforged/fml/loading/FMLPaths.java b/loader/src/main/java/net/neoforged/fml/loading/FMLPaths.java index 180797dd0..165fe5c95 100644 --- a/loader/src/main/java/net/neoforged/fml/loading/FMLPaths.java +++ b/loader/src/main/java/net/neoforged/fml/loading/FMLPaths.java @@ -21,6 +21,7 @@ public enum FMLPaths { GAMEDIR(), MODSDIR("mods"), CONFIGDIR("config"), + CACHEDIR(".fml/cache"), FMLCONFIG(false, CONFIGDIR, "fml.toml"); private static final Logger LOGGER = LogUtils.getLogger(); @@ -89,4 +90,8 @@ public Path relative() { public Path get() { return absolutePath; } + + public Path resolve(String path) { + return absolutePath.resolve(path); + } } diff --git a/loader/src/main/java/net/neoforged/fml/loading/moddiscovery/ModFile.java b/loader/src/main/java/net/neoforged/fml/loading/moddiscovery/ModFile.java index 388d0834b..3a284547e 100644 --- a/loader/src/main/java/net/neoforged/fml/loading/moddiscovery/ModFile.java +++ b/loader/src/main/java/net/neoforged/fml/loading/moddiscovery/ModFile.java @@ -6,10 +6,12 @@ package net.neoforged.fml.loading.moddiscovery; import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; import com.mojang.logging.LogUtils; import cpw.mods.jarhandling.SecureJar; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -37,11 +39,13 @@ import org.apache.maven.artifact.versioning.ArtifactVersion; import org.apache.maven.artifact.versioning.DefaultArtifactVersion; import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @ApiStatus.Internal public class ModFile implements IModFile { private static final Logger LOGGER = LogUtils.getLogger(); + private static final String[] EMPTY_ARRAY = new String[0]; private final String jarVersion; private final ModFileInfoParser parser; @@ -62,14 +66,20 @@ public class ModFile implements IModFile { public static final Attributes.Name TYPE = new Attributes.Name("FMLModType"); private SecureJar.Status securityStatus; - public ModFile(SecureJar jar, final ModFileInfoParser parser, ModFileDiscoveryAttributes attributes) { - this(jar, parser, parseType(jar), attributes); + private final String[] cacheKeyComponents; + + private volatile boolean cacheComputed; + private String cacheKey; + + public ModFile(SecureJar jar, final ModFileInfoParser parser, ModFileDiscoveryAttributes attributes, String... cacheKeyComponents) { + this(jar, parser, parseType(jar), attributes, cacheKeyComponents); } - public ModFile(SecureJar jar, ModFileInfoParser parser, Type type, ModFileDiscoveryAttributes discoveryAttributes) { + public ModFile(SecureJar jar, ModFileInfoParser parser, Type type, ModFileDiscoveryAttributes discoveryAttributes, String... cacheKeyComponents) { this.jar = Objects.requireNonNull(jar, "jar"); this.parser = Objects.requireNonNull(parser, "parser"); this.discoveryAttributes = Objects.requireNonNull(discoveryAttributes, "discoveryAttributes"); + this.cacheKeyComponents = Objects.requireNonNull(cacheKeyComponents, "cache key components").length == 0 ? EMPTY_ARRAY : cacheKeyComponents; manifest = this.jar.moduleDataProvider().getManifest(); modFileType = Objects.requireNonNull(type, "type"); @@ -77,6 +87,29 @@ public ModFile(SecureJar jar, ModFileInfoParser parser, Type type, ModFileDiscov this.modFileInfo = ModFileParser.readModList(this, this.parser); } + @Nullable + public String getCacheKey() { + if (!cacheComputed) { + synchronized (this) { + if (this.cacheKey == null && Files.isRegularFile(jar.getPrimaryPath())) { + try { + var hasher = Hashing.sha256().newHasher() + .putBytes(Files.readAllBytes(jar.getPrimaryPath())); + for (int i = 0; i < cacheKeyComponents.length; i++) { + hasher.putString(cacheKeyComponents[i], StandardCharsets.UTF_8); + } + this.cacheKey = hasher.hash().toString(); + + cacheComputed = true; + } catch (Exception exception) { + throw new RuntimeException(exception); + } + } + } + } + return cacheKey; + } + @Override public Supplier> getSubstitutionMap() { return () -> ImmutableMap.builder().put("jarVersion", jarVersion).putAll(fileProperties).build(); @@ -143,7 +176,7 @@ public List getMixinConfigs() { * Run in an executor thread to harvest the class and annotation list */ public ModFileScanData compileContent() { - return new Scanner(this).scan(); + return new Scanner(this).scanCached(); } public void scanFile(Consumer pathConsumer) { diff --git a/loader/src/main/java/net/neoforged/fml/loading/modscan/Scanner.java b/loader/src/main/java/net/neoforged/fml/loading/modscan/Scanner.java index d395a263a..f3cf481cc 100644 --- a/loader/src/main/java/net/neoforged/fml/loading/modscan/Scanner.java +++ b/loader/src/main/java/net/neoforged/fml/loading/modscan/Scanner.java @@ -6,16 +6,20 @@ package net.neoforged.fml.loading.modscan; import com.mojang.logging.LogUtils; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; +import net.neoforged.fml.loading.FMLPaths; import net.neoforged.fml.loading.LogMarkers; import net.neoforged.fml.loading.moddiscovery.ModFile; import net.neoforged.neoforgespi.language.ModFileScanData; import org.objectweb.asm.ClassReader; import org.slf4j.Logger; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + public class Scanner { private static final Logger LOGGER = LogUtils.getLogger(); private final ModFile fileToScan; @@ -24,6 +28,29 @@ public Scanner(final ModFile fileToScan) { this.fileToScan = fileToScan; } + public ModFileScanData scanCached() { + var key = fileToScan.getCacheKey(); + if (key == null) return scan(); + var path = FMLPaths.CACHEDIR.resolve(key); + try (var in = new ObjectInputStream(Files.newInputStream(path))) { + var scan = ModFileScanData.read(in); + LOGGER.debug("Reading scan data for file {} from cache at {}", fileToScan, path); + if (scan != null) { + scan.addModFileInfo(fileToScan.getModFileInfo()); + return scan; + } + } catch (Exception exception) { + LOGGER.error("Failed to read mod file scan data for file {} from cache at {}", fileToScan, path); + } + var computed = scan(); + try (var out = new ObjectOutputStream(Files.newOutputStream(path))) { + computed.write(out); + } catch (Exception exception) { + LOGGER.error("Failed to write mod file scan data for file {} to cache at {}", fileToScan, path); + } + return computed; + } + public ModFileScanData scan() { ModFileScanData result = new ModFileScanData(); result.addModFileInfo(fileToScan.getModFileInfo()); diff --git a/loader/src/main/java/net/neoforged/neoforgespi/language/ModFileScanData.java b/loader/src/main/java/net/neoforged/neoforgespi/language/ModFileScanData.java index 06a287f4c..a483f355d 100644 --- a/loader/src/main/java/net/neoforged/neoforgespi/language/ModFileScanData.java +++ b/loader/src/main/java/net/neoforged/neoforgespi/language/ModFileScanData.java @@ -5,19 +5,35 @@ package net.neoforged.neoforgespi.language; +import net.neoforged.fml.loading.modscan.ModAnnotation; +import org.objectweb.asm.Type; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.lang.annotation.Annotation; import java.lang.annotation.ElementType; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.IntFunction; import java.util.stream.Stream; -import org.objectweb.asm.Type; public class ModFileScanData { - private final Set annotations = new LinkedHashSet<>(); - private final Set classes = new LinkedHashSet<>(); + private final Set annotations; + private final Set classes; + public ModFileScanData(Set annotations, Set classes) { + this.annotations = annotations; + this.classes = classes; + } + public ModFileScanData() { + this(new LinkedHashSet<>(), new LinkedHashSet<>()); + } private final List modFiles = new ArrayList<>(); public Set getClasses() { @@ -45,4 +61,266 @@ public List getIModInfoData() { public record ClassData(Type clazz, Type parent, Set interfaces) {} public record AnnotationData(Type annotationType, ElementType targetType, Type clazz, String memberName, Map annotationData) {} + + public void write(ObjectOutputStream stream) throws IOException { + stream.writeInt(1); + writeCollection(stream, classes.size(), classes, (st, cls) -> { + st.writeUTF(cls.clazz.getInternalName()); + st.writeUTF(cls.parent.getInternalName()); + writeCollection(st, cls.interfaces.size(), cls.interfaces, (st1, i) -> st1.writeUTF(i.getInternalName())); + }); + writeCollection(stream, annotations.size(), annotations, (st, ann) -> { + st.writeUTF(ann.annotationType().getInternalName()); + st.writeByte((byte)ann.targetType().ordinal()); + st.writeUTF(ann.clazz().getInternalName()); + st.writeUTF(ann.memberName()); + writeCollection(st, ann.annotationData.size(), ann.annotationData.entrySet(), (st1, value) -> { + st1.writeUTF(value.getKey()); + encodeNested(st1, value.getValue()); + }); + }); + } + + @Nullable + public static ModFileScanData read(ObjectInputStream stream) throws IOException { + int spec = stream.readInt(); + if (spec != 1) return null; + + var classesCount = stream.readInt(); + var classes = new LinkedHashSet(classesCount); + while (classesCount > 0) { + var type = Type.getObjectType(stream.readUTF()); + var parent = Type.getObjectType(stream.readUTF()); + var interfaces = readCollection(stream, size -> new HashSet(size), (s, set) -> set.add(Type.getObjectType(s.readUTF()))); + classes.add(new ClassData(type, parent, interfaces)); + classesCount--; + } + + var annotationCount = stream.readInt(); + var annotations = new LinkedHashSet(annotationCount); + while (annotationCount > 0) { + annotations.add(new AnnotationData( + Type.getObjectType(stream.readUTF()), + ElementType.values()[stream.readByte()], + Type.getObjectType(stream.readUTF()), + stream.readUTF(), + readCollection( + stream, + HashMap::new, + (st, map) -> map.put(st.readUTF(), decodeNested(st)) + ) + )); + annotationCount--; + } + + return new ModFileScanData(annotations, classes); + } + + private static C readCollection(ObjectInputStream stream, IntFunction factory, IOExceptionConsumer reader) throws IOException { + var size = stream.readInt(); + var col = factory.apply(size); + while (size > 0) { + reader.consume(stream, col); + size--; + } + return col; + } + + private static void writeCollection(ObjectOutputStream stream, int size, Iterable iterable, IOExceptionConsumer writer) throws IOException { + stream.writeInt(size); + for (T t : iterable) { + writer.consume(stream, t); + } + } + + private static Object decodeNested(ObjectInputStream stream) throws IOException { + return switch (stream.readByte()) { + case 0 -> stream.readUTF(); + case 1 -> stream.readByte(); + case 2 -> stream.readBoolean(); + case 3 -> stream.readShort(); + case 4 -> stream.readChar(); + case 5 -> stream.readInt(); + case 6 -> stream.readLong(); + case 7 -> stream.readFloat(); + case 8 -> stream.readDouble(); + case 9 -> Type.getType(stream.readUTF()); + case 10 -> new ModAnnotation.EnumHolder(stream.readUTF(), stream.readUTF()); + case 11 -> readCollection(stream, ArrayList::new, (st, l) -> l.add(decodeNested(st))); + case 12 -> readCollection(stream, HashMap::new, (st, m) -> m.put(st.readUTF(), decodeNested(st))); + case 13 -> { + var size = stream.readInt(); + var ar = new byte[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readByte(); + yield ar; + } + case 14 -> { + var size = stream.readInt(); + var ar = new boolean[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readBoolean(); + yield ar; + } + case 15 -> { + var size = stream.readInt(); + var ar = new short[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readShort(); + yield ar; + } + case 16 -> { + var size = stream.readInt(); + var ar = new char[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readChar(); + yield ar; + } + case 17 -> { + var size = stream.readInt(); + var ar = new int[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readInt(); + yield ar; + } + case 18 -> { + var size = stream.readInt(); + var ar = new long[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readLong(); + yield ar; + } + case 19 -> { + var size = stream.readInt(); + var ar = new float[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readFloat(); + yield ar; + } + case 20 -> { + var size = stream.readInt(); + var ar = new double[size]; + for (int i = 0; i < size; i++) ar[i] = stream.readDouble(); + yield ar; + } + default -> throw new IllegalArgumentException(); + }; + } + + @SuppressWarnings("ForLoopReplaceableByForEach") // an indexed for loop is a bit faster and more memory efficient + private static void encodeNested(ObjectOutputStream stream, Object object) throws IOException { + switch (object) { + case String st -> { + stream.writeByte(0); + stream.writeUTF(st); + } + case Byte i -> { + stream.writeByte(1); + stream.writeByte(i); + } + case Boolean i -> { + stream.writeByte(2); + stream.writeBoolean(i); + } + case Short i -> { + stream.writeByte(3); + stream.writeShort(i); + } + case Character i -> { + stream.writeByte(4); + stream.writeChar(i); + } + case Integer i -> { + stream.writeByte(5); + stream.writeInt(i); + } + case Long i -> { + stream.writeByte(6); + stream.writeLong(i); + } + case Float i -> { + stream.writeByte(7); + stream.writeFloat(i); + } + case Double i -> { + stream.writeByte(8); + stream.writeDouble(i); + } + case Type t -> { + stream.writeByte(9); + stream.writeUTF(t.getDescriptor()); + } + case ModAnnotation.EnumHolder(var desc, var val) -> { + stream.writeByte(10); + stream.writeUTF(desc); + stream.writeUTF(val); + } + case List list -> { + stream.writeByte(11); + writeCollection(stream, list.size(), list, ModFileScanData::encodeNested); + } + case Map map -> { + stream.writeByte(12); + writeCollection(stream, map.size(), map.entrySet(), (st1, in) -> { + st1.writeUTF((String)in.getKey()); + encodeNested(st1, in.getValue()); + }); + } + case byte[] b -> { + stream.writeInt(13); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeByte(b[i]); + } + } + case boolean[] b -> { + stream.writeInt(14); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeBoolean(b[i]); + } + } + case short[] b -> { + stream.writeInt(15); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeShort(b[i]); + } + } + case char[] b -> { + stream.writeInt(16); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeChar(b[i]); + } + } + case int[] b -> { + stream.writeInt(17); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeInt(b[i]); + } + } + case long[] b -> { + stream.writeInt(18); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeLong(b[i]); + } + } + case double[] b -> { + stream.writeInt(19); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeDouble(b[i]); + } + } + case float[] b -> { + stream.writeInt(20); + stream.writeInt(b.length); + for (int i = 0; i < b.length; i++) { + stream.writeFloat(b[i]); + } + } + default -> throw new IllegalArgumentException(); + } + } + + @FunctionalInterface + private interface IOExceptionConsumer { + void consume(F stream, I input) throws IOException; + } } diff --git a/loader/src/test/java/net/neoforged/fml/test/TestModFile.java b/loader/src/test/java/net/neoforged/fml/test/TestModFile.java index 9c3e17ac6..760ba2f64 100644 --- a/loader/src/test/java/net/neoforged/fml/test/TestModFile.java +++ b/loader/src/test/java/net/neoforged/fml/test/TestModFile.java @@ -9,6 +9,7 @@ import com.electronwill.nightconfig.toml.TomlParser; import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import cpw.mods.jarhandling.JarContentsBuilder; import cpw.mods.jarhandling.SecureJar; @@ -19,6 +20,7 @@ import net.neoforged.fml.loading.moddiscovery.ModJarMetadata; import net.neoforged.fml.loading.moddiscovery.NightConfigWrapper; import net.neoforged.fml.loading.modscan.Scanner; +import net.neoforged.neoforgespi.language.ModFileScanData; import net.neoforged.neoforgespi.locating.ModFileDiscoveryAttributes; import net.neoforged.neoforgespi.locating.ModFileInfoParser; import org.intellij.lang.annotations.Language; @@ -51,8 +53,10 @@ public RuntimeCompiler.CompilationBuilder classBuilder() { return compiler.builder(); } - public void scan() { + @CanIgnoreReturnValue + public ModFileScanData scan() { setScanResult(new Scanner(this).scan(), null); + return getScanResult(); } @CheckReturnValue diff --git a/loader/src/test/java/net/neoforged/neoforgespi/language/ScanDataTest.java b/loader/src/test/java/net/neoforged/neoforgespi/language/ScanDataTest.java index cd78ec7b4..fdfde66c5 100644 --- a/loader/src/test/java/net/neoforged/neoforgespi/language/ScanDataTest.java +++ b/loader/src/test/java/net/neoforged/neoforgespi/language/ScanDataTest.java @@ -6,16 +6,45 @@ package net.neoforged.neoforgespi.language; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.lang.annotation.ElementType; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; + +import cpw.mods.jarhandling.SecureJar; +import net.bytebuddy.agent.ByteBuddyAgent; +import net.neoforged.fml.loading.LauncherTest; +import net.neoforged.fml.loading.moddiscovery.ModFile; +import net.neoforged.fml.loading.moddiscovery.readers.JarModsDotTomlModFileReader; import net.neoforged.fml.loading.modscan.ModAnnotation; +import net.neoforged.fml.loading.modscan.Scanner; import net.neoforged.fml.test.TestModFile; +import net.neoforged.neoforgespi.locating.ModFileDiscoveryAttributes; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.objectweb.asm.Type; public class ScanDataTest { + @BeforeAll + static void ensureAddOpensForUnionFs() { + // We abuse the ByteBuddy agent that Mockito also uses to open java.lang to UnionFS + var instrumentation = ByteBuddyAgent.install(); + instrumentation.redefineModule( + MethodHandles.class.getModule(), + Set.of(), + Map.of(), + Map.of("java.lang.invoke", Set.of(LauncherTest.class.getModule())), + Set.of(), + Map.of()); + } + @Test void testSimpleAnnotations() throws IOException { try (final var mod = modFile()) { @@ -162,6 +191,47 @@ class FieldTest { } } + @Test + void testRoundTrip(@TempDir Path temp) throws IOException { + var path = temp.resolve("metadata"); + + try (var mod = modFile()) { + mod.classBuilder() + .addClass("com.example.Test", """ + @interface SomeAnn { + } + + class FieldTest { + @SomeAnn + public int counter; + }""") + .addClass("com.anotherexample.Test2", """ + @Deprecated + public class Test2 implements Runnable { + @Override public void run() {} + }""") + .compile(); + var res = mod.scan(); + + try (var out = new ObjectOutputStream(Files.newOutputStream(path))) { + res.write(out); + } + + ModFileScanData newRes; + try (var in = new ObjectInputStream(Files.newInputStream(path))) { + newRes = ModFileScanData.read(in); + } + + Assertions.assertThat(newRes).isNotNull(); + + Assertions.assertThat(res.getAnnotations()) + .isEqualTo(newRes.getAnnotations()); + + Assertions.assertThat(res.getClasses()) + .isEqualTo(newRes.getClasses()); + } + } + private static TestModFile modFile() { return TestModFile.newInstance(""" modLoader="javafml"