A bit of experimentation with APIs

.editorconfig

@@ -0,0 +1,17 @@
+[*]
+charset = utf-8
+end_of_line = crlf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+tab_width = 4
+ij_continuation_indent_size = 4
+
+[*.java]
+ij_java_do_while_brace_force = always
+ij_java_for_brace_force = always
+ij_java_if_brace_force = always
+ij_java_while_brace_force = always
+ij_java_call_parameters_new_line_after_left_paren = true
+ij_java_call_parameters_right_paren_on_new_line = true
+ij_java_call_parameters_wrap = off

build.gradle

@@ -57,4 +57,4 @@ allprojects {
         testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.1'
         testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.1'
     }
-}
+}

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/DirectStorageReaderTest.java

@@ -1,62 +1,73 @@
 package com.shade.decima.game.hrzr;
 
-import com.shade.decima.game.Asset;
-import com.shade.decima.game.AssetId;
-import com.shade.decima.game.hrzr.rtti.HRZRTypeFactory;
-import com.shade.decima.game.hrzr.rtti.HRZRTypeReader;
+import com.shade.decima.game.FileSystem;
+import com.shade.decima.game.hrzr.rtti.HorizonTypeFactory;
+import com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered;
+import com.shade.decima.game.hrzr.storage.HorizonAssetId;
+import com.shade.decima.game.hrzr.storage.HorizonAssetManager;
 import com.shade.decima.game.hrzr.storage.PackFileManager;
-import com.shade.decima.game.hrzr.storage.PathResolver;
-import com.shade.decima.rtti.factory.TypeNotFoundException;
+import com.shade.decima.rtti.data.Ref;
 import com.shade.util.NotNull;
-import com.shade.util.io.BinaryReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.List;
+import java.util.Iterator;
+import java.util.function.Function;
 
 import static com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.ERenderPlatform;
 
 public class DirectStorageReaderTest {
     private static final Logger log = LoggerFactory.getLogger(DirectStorageReaderTest.class);
 
     public static void main(String[] args) throws IOException {
-        var source = Path.of("E:/SteamLibrary/steamapps/common/Horizon Zero Dawn Remastered");
-        var resolver = new HorizonPathResolver(source);
+        var fileSystem = new HorizonFileSystem(Path.of("E:/SteamLibrary/steamapps/common/Horizon Zero Dawn Remastered"));
+        var typeFactory = new HorizonTypeFactory();
 
         log.info("Loading archives");
-        try (var manager = new PackFileManager(resolver)) {
-            var factory = new HRZRTypeFactory();
-            var reader = new HRZRTypeReader();
-            var assets = new HashMap<AssetId, Asset>();
+        try (var packFileManager = new PackFileManager(fileSystem)) {
+            var assetManager = new HorizonAssetManager(packFileManager, typeFactory);
+            var asset = assetManager.get(
+                HorizonAssetId.ofPath("levels/game.core", "142071c5-7493-37d8-af34-361249017a88"),
+                HorizonZeroDawnRemastered.Game.class
+            );
 
-            for (Asset asset : manager.assets()) {
-                assets.put(asset.id(), asset);
+            for (var levelGroup : deref(asset.general().levelGroups())) {
+                log.debug("{}", levelGroup.general().name());
+                for (var level : deref(levelGroup.general().levels())) {
+                    log.debug("  {} - {}", level.general().name(), level.general().levelData());
+                }
             }
+        }
+    }
 
-            var slice = assets.values();
-            var index = 0;
+    @NotNull
+    private static <T> Iterable<T> deref(@NotNull Iterable<Ref<T>> iterable) {
+        return map(iterable, Ref::get);
+    }
 
-            log.info("Reading {} assets", slice.size());
-            for (Asset asset : slice) {
-                var id = asset.id();
-                var data = BinaryReader.wrap(manager.load(id));
+    @NotNull
+    private static <T, R> Iterable<R> map(@NotNull Iterable<T> iterable, @NotNull Function<T, R> mapper) {
+        return () -> map(iterable.iterator(), mapper);
+    }
 
-                try {
-                    List<Object> objects = reader.read(data, factory);
-                    log.info("[{}/{}] Read {} objects", index, slice.size(), objects.size());
-                } catch (TypeNotFoundException e) {
-                    log.error("[{}/{}] Unable to read: {}", index, slice.size(), e.getMessage());
-                }
+    @NotNull
+    private static <T, R> Iterator<R> map(@NotNull Iterator<T> iterator, @NotNull Function<T, R> mapper) {
+        return new Iterator<>() {
+            @Override
+            public boolean hasNext() {
+                return iterator.hasNext();
+            }
 
-                index++;
+            @Override
+            public R next() {
+                return mapper.apply(iterator.next());
             }
-        }
+        };
     }
 
-    private record HorizonPathResolver(@NotNull Path source) implements PathResolver {
+    private record HorizonFileSystem(@NotNull Path source) implements FileSystem {
         @NotNull
         @Override
         public Path resolve(@NotNull String path) {

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/PackFileDevice.java

@@ -1,6 +1,6 @@
 package com.shade.decima.game.hrzr;
 
-import com.shade.decima.game.hrzr.storage.PathResolver;
+import com.shade.decima.game.FileSystem;
 import com.shade.util.NotNull;
 import com.shade.util.io.BinaryReader;
 import com.shade.util.io.DirectStorageReader;
@@ -18,9 +18,9 @@ public class PackFileDevice implements Closeable {
     private static final Logger log = LoggerFactory.getLogger(PackFileDevice.class);
 
     private final Map<String, BinaryReader> files = new HashMap<>();
-    private final PathResolver resolver;
+    private final FileSystem resolver;
 
-    public PackFileDevice(@NotNull PathResolver resolver) {
+    public PackFileDevice(@NotNull FileSystem resolver) {
         this.resolver = resolver;
     }
 

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/rtti/HRZRTypeReader.java

@@ -24,7 +24,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.*;
+import static com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.GGUUID;
+import static com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.RTTIRefObject;
 
 public class HRZRTypeReader extends AbstractTypeReader {
     private final List<Ref<?>> pointers = new ArrayList<>();
@@ -273,6 +274,11 @@ private record StreamingRef<T>(@NotNull GGUUID objectUUID, @NotNull String filen
         public T get() {
             throw new NotImplementedException();
         }
+
+        @Override
+        public String toString() {
+            return filename;
+        }
     }
 
     private record UUIDRef<T>(@NotNull GGUUID objectUUID) implements Ref<T> {

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/rtti/HRZRTypeFactory.java → modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/rtti/HorizonTypeFactory.java

@@ -12,8 +12,8 @@
 import java.util.Comparator;
 import java.util.List;
 
-public class HRZRTypeFactory extends AbstractTypeFactory {
-    public HRZRTypeFactory() {
+public class HorizonTypeFactory extends AbstractTypeFactory {
+    public HorizonTypeFactory() {
         super(HorizonZeroDawnRemastered.class, MethodHandles.lookup());
     }
 

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/storage/HorizonAssetId.java

@@ -0,0 +1,16 @@
+package com.shade.decima.game.hrzr.storage;
+
+import com.shade.decima.game.AssetId;
+import com.shade.util.NotNull;
+import com.shade.util.hash.Hashing;
+
+import java.util.Locale;
+import java.util.UUID;
+
+public record HorizonAssetId(long fileHash, @NotNull UUID objectUuid) implements AssetId {
+    public static HorizonAssetId ofPath(@NotNull String path, @NotNull String uuid) {
+        var norm = path.toLowerCase(Locale.ROOT) + '\0';
+        var hash = Hashing.decimaMurmur3().hashString(norm).asLong();
+        return new HorizonAssetId(hash, UUID.fromString(uuid));
+    }
+}

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/storage/HorizonAssetManager.java

@@ -0,0 +1,73 @@
+package com.shade.decima.game.hrzr.storage;
+
+import com.shade.decima.game.AssetId;
+import com.shade.decima.game.AssetManager;
+import com.shade.decima.game.hrzr.rtti.HRZRTypeReader;
+import com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.GGUUID;
+import com.shade.decima.game.hrzr.rtti.HorizonZeroDawnRemastered.RTTIRefObject;
+import com.shade.decima.rtti.factory.TypeFactory;
+import com.shade.util.NotNull;
+import com.shade.util.io.BinaryReader;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import java.util.UUID;
+
+public class HorizonAssetManager implements AssetManager, Closeable {
+    private final PackFileManager packFileManager;
+    private final TypeFactory typeFactory;
+
+    public HorizonAssetManager(@NotNull PackFileManager packFileManager, @NotNull TypeFactory typeFactory) {
+        this.packFileManager = packFileManager;
+        this.typeFactory = typeFactory;
+    }
+
+    @NotNull
+    @Override
+    public <T> T get(@NotNull AssetId id, @NotNull Class<T> type) throws IOException {
+        var assetId = (HorizonAssetId) id;
+        var objectUUID = convertUUID(assetId.objectUuid());
+
+        var buffer = packFileManager.load(PackFileAssetId.ofHash(assetId.fileHash()));
+        var objects = new HRZRTypeReader().read(BinaryReader.wrap(buffer), typeFactory);
+
+        var object = objects.stream()
+            .map(RTTIRefObject.class::cast)
+            .filter(o -> o.general().objectUUID().equals(objectUUID))
+            .findFirst().orElseThrow(() -> new NoSuchElementException("Can't find asset " + id));
+
+        return type.cast(object);
+    }
+
+    @Override
+    public void close() throws IOException {
+        packFileManager.close();
+    }
+
+    @NotNull
+    private GGUUID convertUUID(@NotNull UUID uuid) {
+        var msb = uuid.getMostSignificantBits();
+        var lsb = uuid.getLeastSignificantBits();
+
+        var object = typeFactory.newInstance(GGUUID.class);
+        object.data0((byte) (msb >>> 56));
+        object.data1((byte) (msb >>> 48));
+        object.data2((byte) (msb >>> 40));
+        object.data3((byte) (msb >>> 32));
+        object.data4((byte) (msb >>> 24));
+        object.data5((byte) (msb >>> 16));
+        object.data6((byte) (msb >>> 8));
+        object.data7((byte) (msb));
+        object.data8((byte) (lsb >>> 56));
+        object.data9((byte) (lsb >>> 48));
+        object.data10((byte) (lsb >>> 40));
+        object.data11((byte) (lsb >>> 32));
+        object.data12((byte) (lsb >>> 24));
+        object.data13((byte) (lsb >>> 16));
+        object.data14((byte) (lsb >>> 8));
+        object.data15((byte) (lsb));
+
+        return object;
+    }
+}

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/storage/PackFileManager.java

@@ -1,9 +1,8 @@
 package com.shade.decima.game.hrzr.storage;
 
-import com.shade.decima.game.Asset;
 import com.shade.decima.game.AssetId;
+import com.shade.decima.game.FileSystem;
 import com.shade.util.NotNull;
-import com.shade.util.Nullable;
 import com.shade.util.io.BinaryReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,17 +12,16 @@
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
+import java.util.*;
 
 public class PackFileManager implements Closeable {
     private static final Logger log = LoggerFactory.getLogger(PackFileManager.class);
 
     private final NavigableSet<PackFileArchive> archives = new TreeSet<>();
+    private final Map<PackFileAssetId, PackFileArchive> assets = new HashMap<>();
 
-    public PackFileManager(@NotNull PathResolver resolver) throws IOException {
-        var root = resolver.resolve("cache:package");
+    public PackFileManager(@NotNull FileSystem fileSystem) throws IOException {
+        var root = fileSystem.resolve("cache:package");
 
         try (BinaryReader reader = BinaryReader.open(root.resolve("PackFileLocators.bin"))) {
             var count = reader.readInt();
@@ -33,6 +31,12 @@ public PackFileManager(@NotNull PathResolver resolver) throws IOException {
                 mount(path, info);
             }
         }
+
+        for (PackFileArchive archive : archives.reversed()) {
+            for (PackFileAsset asset : archive.assets()) {
+                assets.putIfAbsent(asset.id(), archive);
+            }
+        }
     }
 
     private void mount(@NotNull Path path, @NotNull PackFileInfo info) throws IOException {
@@ -45,19 +49,16 @@ private void mount(@NotNull Path path, @NotNull PackFileInfo info) throws IOExce
     }
 
     @NotNull
-    public List<? extends Asset> assets() {
-        return archives.stream()
-            .flatMap(archive -> archive.assets().stream())
-            .distinct()
-            .toList();
+    public ByteBuffer load(@NotNull AssetId id) throws IOException {
+        PackFileArchive archive = assets.get((PackFileAssetId) id);
+        if (archive == null) {
+            throw new NoSuchElementException("Asset not found: " + id);
+        }
+        return archive.load(id);
     }
 
-    @NotNull
-    public ByteBuffer load(@NotNull AssetId id) throws IOException {
-        return archives.reversed().stream()
-            .filter(archive -> archive.contains(id))
-            .findFirst().orElseThrow()
-            .load(id);
+    public boolean contains(@NotNull AssetId id) {
+        return assets.containsKey((PackFileAssetId) id);
     }
 
     @Override
@@ -66,6 +67,7 @@ public void close() throws IOException {
             archive.close();
         }
         archives.clear();
+        assets.clear();
     }
 
     record PackFileAssetInfo(long hash, long offset, int length) {

modules/decima-game/src/main/java/com/shade/decima/game/AssetManager.java

@@ -0,0 +1,10 @@
+package com.shade.decima.game;
+
+import com.shade.util.NotNull;
+
+import java.io.IOException;
+
+public interface AssetManager {
+    @NotNull
+    <T> T get(@NotNull AssetId id, @NotNull Class<T> type) throws IOException;
+}

modules/decima-game-horizon-zero-dawn-remastered/src/main/java/com/shade/decima/game/hrzr/storage/PathResolver.java → modules/decima-game/src/main/java/com/shade/decima/game/FileSystem.java

@@ -1,10 +1,10 @@
-package com.shade.decima.game.hrzr.storage;
+package com.shade.decima.game;
 
 import com.shade.util.NotNull;
 
 import java.nio.file.Path;
 
-public interface PathResolver {
+public interface FileSystem {
     @NotNull
     Path resolve(@NotNull String path);
 }