Add copy option for conversion to SectorFile

The copy option does not perform decompression/recompression,
which should make it faster for systems limited by CPU resources.

Additionally, read the entire RegionFile into memory when converting
to SectorFile to avoid random read costs when converting.

README.md

@@ -98,10 +98,16 @@ The compression type to use on the new SectorFiles. Values:
 * NONE (id = 3)
 * LZ4 (id = 4)
 * ZSTD (id = 5)
+* COPY (id = -1)
 
 DEFLATE (id = 2) is the current default. LZ4 (id = 4) is fast but not space 
 efficient. ZSTD (id = 5) is fast and slightly less space efficient than DEFLATE.
 
+The COPY compression type will copy the compressed data from the source 
+RegionFiles. Note that the COPY compression type will not perform 
+decompression/recompression, unlike the rest of the compression options, 
+which should greatly reduce CPU load.
+
 
 ### Conversion from SectorFile to RegionFile (Deconversion)
 

dependency-reduced-pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>ca.spottedleaf</groupId>
   <artifactId>sectortool</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <version>1.1-SNAPSHOT</version>
   <build>
     <defaultGoal>clean package</defaultGoal>
     <plugins>

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>ca.spottedleaf</groupId>
     <artifactId>sectortool</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>1.1-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

src/main/java/ca/spottedleaf/io/region/SectorFile.java

@@ -108,7 +108,7 @@ public static final class FileHeader {
         public final int[] typeHeaderOffsets = new int[MAX_TYPES];
 
         public FileHeader() {
-            if (ABSENT_HEADER_XXHASH64 != 0L || ABSENT_TYPE_HEADER_OFFSET != 0) {
+            if (ABSENT_HEADER_XXHASH64 != 0L || ABSENT_TYPE_HEADER_OFFSET != 0) { // arrays default initialise to 0
                 this.reset();
             }
         }
@@ -896,7 +896,7 @@ private void readFileHeader(final BufferChoices unscopedBufferChoices) throws IO
                 // note: only the type headers can bypass the max limit, as the max limit is determined by SECTOR_OFFSET_BITS
                 //       but the type offset is full 31 bits
                 if (typeHeaderOffset < 0 || !this.sectorAllocator.tryAllocateDirect(typeHeaderOffset, TYPE_HEADER_SECTORS, true)) {
-                    LOGGER.error("File '" + this.file.getAbsolutePath() + "' has bad or overlapping offset for type " + this.debugType(i) + ": " + typeHeaderOffset);
+                    LOGGER.error("File '" + this.file.getAbsolutePath() + "' has bad or overlapping offset for type header " + this.debugType(i) + ": " + typeHeaderOffset);
                     needsRecalculation = true;
                     continue;
                 }
@@ -1283,6 +1283,8 @@ public boolean delete(final BufferChoices unscopedBufferChoices, final int local
 
     // performs a sync as if the sync flag is used for creating the sectorfile
     public static final int WRITE_FLAG_SYNC         = 1 << 0;
+    // avoid compressing data, but store target compression type (use if the data is already compressed)
+    public static final int WRITE_FLAG_RAW          = 1 << 1;
 
     public static record SectorFileOutput(
             /* Must run save (before close()) to cause the data to be written to the file, close() will not do this */
@@ -1306,7 +1308,7 @@ public SectorFileOutput write(final BufferChoices scopedBufferChoices, final int
         final SectorFileOutputStream output = new SectorFileOutputStream(
                 scopedBufferChoices, localX, localZ, type, useCompressionType, writeFlags
         );
-        final OutputStream compressedOut = useCompressionType.createOutput(scopedBufferChoices, output);
+        final OutputStream compressedOut = (writeFlags & WRITE_FLAG_RAW) != 0 ? output : useCompressionType.createOutput(scopedBufferChoices, output);
 
         return new SectorFileOutput(output, new DataOutputStream(compressedOut));
     }
@@ -1489,6 +1491,10 @@ private SectorFileOutputStream(final BufferChoices scopedBufferChoices,
             this.writeFlags = writeFlags;
         }
 
+        public int getTotalCompressedSize() {
+            return this.totalCompressedSize + (this.buffer == null ? 0 : this.buffer.position());
+        }
+
         @Override
         protected ByteBuffer flush(final ByteBuffer current) throws IOException {
             if (this.externalFile == null && current.hasRemaining()) {
@@ -1556,7 +1562,7 @@ private void save() throws IOException {
                 buffer.flip();
 
                 final long dataHash = XXHASH64.hash(
-                        this.buffer, DataHeader.DATA_HEADER_LENGTH, buffer.remaining() - DataHeader.DATA_HEADER_LENGTH,
+                        buffer, DataHeader.DATA_HEADER_LENGTH, buffer.remaining() - DataHeader.DATA_HEADER_LENGTH,
                         XXHASH_SEED
                 );
 
@@ -1609,7 +1615,9 @@ public void flush() throws IOException {
             return;
         }
         if (!this.readOnly) {
-            this.channel.force(true);
+            if (this.sync) {
+                this.channel.force(true);
+            }
         }
     }
 

src/main/java/ca/spottedleaf/io/region/SectorFileCompressionType.java

@@ -106,4 +106,24 @@ public final int getId() {
     public static SectorFileCompressionType getById(final int id) {
         return BY_ID.get(id);
     }
+
+    public static SectorFileCompressionType fromRegionFile(final int id) {
+        switch (id) {
+            case 1: { // GZIP
+                return SectorFileCompressionType.GZIP;
+            }
+            case 2: { // DEFLATE
+                return SectorFileCompressionType.DEFLATE;
+            }
+            case 3: { // NONE
+                return SectorFileCompressionType.NONE;
+            }
+            case 4: { // LZ4
+                return SectorFileCompressionType.LZ4;
+            }
+            default: {
+                throw new IllegalArgumentException();
+            }
+        }
+    }
 }

src/main/java/ca/spottedleaf/sectortool/conversion/ConvertWorld.java

@@ -31,6 +31,8 @@ public final class ConvertWorld {
     public static final String INPUT_PROPERTY = "input";
     public static final String TYPE_OUTPUT_PROPERTY = "compress";
 
+    public static final int COMPRESSION_TYPE_COPY_ID = -1;
+
     private static ExecutorService createExecutors() {
         return Executors.newFixedThreadPool(Main.THREADS, new ThreadFactory() {
             private final AtomicInteger idGenerator = new AtomicInteger();
@@ -62,6 +64,7 @@ private static void decrementTracker(final AtomicInteger concurrentTracker, fina
     private static void submitToExecutor(final ExecutorService executor, final File dimDirectory, final String regionName,
                                          final SectorFileCompressionType compressionType, final BufferChoices unscopedBufferChoices,
                                          final AtomicInteger concurrentTracker, final Thread wakeup, final int threshold) {
+        final boolean raw = compressionType == null;
         // old format is r.<x>.<z>.mca
 
         final String[] coords = regionName.substring(2, regionName.length() - RegionFile.ANVIL_EXTENSION.length()).split("\\.");
@@ -85,7 +88,7 @@ private static void submitToExecutor(final ExecutorService executor, final File
         final SectorFile outputFile;
         try {
             outputFile = new SectorFile(
-                    output, sectionX, sectionZ, compressionType, unscopedBufferChoices,
+                    output, sectionX, sectionZ, raw ? SectorFileCompressionType.NONE : compressionType, unscopedBufferChoices,
                     MinecraftRegionFileType.getTranslationTable(),
                     0
             );
@@ -123,6 +126,19 @@ class RegionFileReader implements Runnable {
 
             @Override
             public void run() {
+                try (final BufferChoices readScope = unscopedBufferChoices.scope()) {
+                    for (final RegionFile regionFile : byId.values()) {
+                        try {
+                            regionFile.fillRaw(readScope);
+                        } catch (final IOException ex) {
+                            synchronized (System.err) {
+                                System.err.println("Failed to read raw from regionfile " + regionFile.file.getAbsolutePath());
+                                ex.printStackTrace(System.err);
+                            }
+                        }
+                    }
+                }
+
                 try (final BufferChoices readScope = unscopedBufferChoices.scope()) {
                     final RegionFile.CustomByteArrayOutputStream decompressed = new RegionFile.CustomByteArrayOutputStream(readScope.t1m().acquireJavaBuffer());
 
@@ -138,9 +154,9 @@ public void run() {
 
                             decompressed.reset();
 
-                            boolean read = false;
+                            int read = -1;
                             try {
-                                read = regionFile.read(chunkX, chunkZ, readScope, decompressed);
+                                read = regionFile.read(chunkX, chunkZ, readScope, decompressed, raw);
                             } catch (final IOException ex) {
                                 synchronized (System.err) {
                                     System.err.println(
@@ -150,13 +166,15 @@ public void run() {
                                 }
                             }
 
-                            if (!read) {
+                            if (read < 0) {
                                 continue;
                             }
 
                             try (final BufferChoices writeScope = readScope.scope();) {
                                final SectorFile.SectorFileOutput output = outputFile.write(
-                                       writeScope, chunkX, chunkZ, type.getNewId(), null, 0
+                                       writeScope, chunkX, chunkZ, type.getNewId(),
+                                       raw ? SectorFileCompressionType.fromRegionFile(read) : null,
+                                       (raw ? SectorFile.WRITE_FLAG_RAW : 0)
                                );
                                try {
                                    decompressed.writeTo(output.outputStream());
@@ -216,8 +234,9 @@ public static void run(final String[] args) {
             return;
         }
 
-        final SectorFileCompressionType compressionType = SectorFileCompressionType.getById(Integer.getInteger(TYPE_OUTPUT_PROPERTY, -1));
-        if (compressionType == null) {
+        final int compressionTypeId = Integer.getInteger(TYPE_OUTPUT_PROPERTY, -1);
+        final SectorFileCompressionType compressionType = SectorFileCompressionType.getById(compressionTypeId);
+        if (compressionType == null && compressionTypeId != COMPRESSION_TYPE_COPY_ID) {
             System.err.println("Specified compression type is absent (-D" + TYPE_OUTPUT_PROPERTY + "=<id>) or invalid");
             System.err.println("Select one from: 1 (GZIP), 2 (DEFLATE), 3 (NONE), 4 (LZ4), 5 (ZSTD)");
             return;