Merge pull request #23 from rtbo/zip_data_descriptor

Zip data descriptor
rtbo · Sep 26, 2023 · 6a10caa · 6a10caa
2 parents 67de9a3 + be32c0e
commit 6a10caa
Show file tree

Hide file tree

Showing 4 changed files with 122 additions and 7 deletions.
diff --git a/src/squiz_box/box/zip.d b/src/squiz_box/box/zip.d
@@ -438,6 +438,8 @@ private struct ZipUnbox(C) if (is(C : Cursor))
     ulong nextHeader;
     Flag!"removePrefix" removePrefix;
     string prefix;
+    bool dataDescriptor;
+    bool zip64;
 
     static if (isSearchable)
     {
@@ -634,27 +636,26 @@ private struct ZipUnbox(C) if (is(C : Cursor))
             (flag & ZipFlag.encryption) == ZipFlag.none,
             "Zip encryption unsupported"
         );
-        enforce(
-            (flag & ZipFlag.dataDescriptor) == ZipFlag.none,
-            "Zip format unsupported (data descriptor)"
-        );
         enforce(
             header.compressionMethod.val == 0 || header.compressionMethod.val == 8,
             "Unsupported Zip compression method"
         );
 
+        info.dataDescriptor = (flag & ZipFlag.dataDescriptor) != ZipFlag.none;
         info.deflated = header.compressionMethod.val == 8;
         info.expectedCrc32 = header.crc32.val;
 
         if (efInfo.has(KnownExtraField.zip64))
         {
             info.size = efInfo.uncompressedSize;
             info.compressedSize = efInfo.compressedSize;
+            info.zip64 = true;
         }
         else
         {
             info.size = header.uncompressedSize.val;
             info.compressedSize = header.compressedSize.val;
+            info.zip64 = false;
         }
 
         info.type = info.compressedSize == 0 ? EntryType.directory : EntryType.regular;
@@ -691,6 +692,13 @@ private struct ZipUnbox(C) if (is(C : Cursor))
     {
         import std.datetime.systime : DosFileTimeToSysTime, unixTimeToStdTime, SysTime;
 
+        if (dataDescriptor)
+        {
+            ZipDataDescriptor.read(input, zip64);
+            dataDescriptor = false;
+            zip64 = false;
+        }
+
         LocalFileHeader header = void;
         input.readValue(&header);
         if (header.signature == CentralFileHeader.expectedSignature)
@@ -724,6 +732,10 @@ private struct ZipUnbox(C) if (is(C : Cursor))
             ZipEntryInfo info;
             info.path = path;
             fillEntryInfo(info, efInfo, header);
+            enforce(!info.dataDescriptor,
+                "ZIP files with data descriptor can only be read with seekable data source." ~
+                " Try to call `unboxZip` with a `File` or `ubyte[]` input parameter."
+            );
             // educated guess for the size in the central directory
             info.entrySize = header.totalLength() +
                 info.compressedSize +
@@ -757,6 +769,8 @@ private struct ZipUnbox(C) if (is(C : Cursor))
                 readEntry();
         }
 
+        dataDescriptor = info.dataDescriptor;
+        zip64 = info.zip64;
         currentEntry = new ZipUnboxEntry!C(input, info);
     }
 }
@@ -806,6 +820,8 @@ private struct ZipEntryInfo
     uint attributes;
     bool deflated;
     uint expectedCrc32;
+    bool dataDescriptor;
+    bool zip64;
 
     version (Posix)
     {
@@ -814,6 +830,67 @@ private struct ZipEntryInfo
     }
 }
 
+private struct ZipDataDescriptor
+{
+    enum signature = 0x08074b50;
+
+    uint crc32;
+    ulong compressedSize;
+    ulong uncompressedSize;
+
+    static ZipDataDescriptor read(Cursor cursor, bool zip64)
+    {
+        ZipDataDescriptor res = void;
+
+        // the signature is optional (but what if the CRC32 actually equals the signature code ??)
+        auto crc32 = cursor.getValue!(LittleEndian!4)();
+        if (crc32.val == signature)
+        {
+            crc32 = cursor.getValue!(LittleEndian!4)();
+        }
+        res.crc32 = crc32.val;
+
+        if (zip64)
+        {
+            res.compressedSize = cursor.getValue!(LittleEndian!8)().val;
+            res.uncompressedSize = cursor.getValue!(LittleEndian!8)().val;
+        }
+        else
+        {
+            res.compressedSize = cursor.getValue!(LittleEndian!4)().val;
+            res.uncompressedSize = cursor.getValue!(LittleEndian!4)().val;
+        }
+
+        return res;
+    }
+
+    void write(WriteCursor cursor, bool zip64)
+    {
+        assert((uncompressedSize <= 0xffffffff && compressedSize == 0xffffffff) | zip64);
+
+        LittleEndian!4 b4 = signature;
+        cursor.putValue(b4);
+        b4 = this.crc32;
+        cursor.putValue(b4);
+
+        if (zip64)
+        {
+            LittleEndian!8 b8 = this.compressedSize;
+            cursor.putValue(b8);
+            b8 = this.uncompressedSize;
+            cursor.putValue(b8);
+        }
+        else
+        {
+            b4 = cast(uint)this.compressedSize;
+            cursor.putValue(b4);
+            b4 = cast(uint)this.uncompressedSize;
+            cursor.putValue(b4);
+        }
+    }
+
+}
+
 private enum KnownExtraField
 {
     none = 0,

diff --git a/src/squiz_box/priv.d b/src/squiz_box/priv.d
@@ -426,7 +426,14 @@ interface WriteCursor
 {
     void put(ubyte val);
 
-    void write(const(ubyte)[] arr);
+    void write(scope const(ubyte)[] arr);
+
+    void putValue(T)(T value)
+    {
+        scope ubyte* ptr = cast(ubyte*)(&value);
+        scope ubyte[] arr = ptr[0 .. T.sizeof];
+        write(arr);
+    }
 }
 
 interface SearchableWriteCursor : WriteCursor
@@ -490,7 +497,7 @@ class ArrayWriteCursor : SearchableWriteCursor
         _writePos++;
     }
 
-    void write(const(ubyte)[] arr)
+    void write(scope const(ubyte)[] arr)
     {
         if (_writePos == _data.length)
         {
@@ -550,7 +557,7 @@ class FileWriteCursor : WriteCursor
         _file.rawWrite(buf);
     }
 
-    void write(const(ubyte)[] arr)
+    void write(scope const(ubyte)[] arr)
     {
         _file.rawWrite(arr);
     }

diff --git a/test/archive.d b/test/archive.d
@@ -364,6 +364,37 @@ unittest
     testExtractedFiles(dm, Yes.mode666);
 }
 
+@("Extract Zip With data descriptor")
+unittest
+{
+    import std.algorithm : canFind, map;
+    import std.array : array;
+    import std.exception : assertThrown;
+    import std.file : mkdir;
+    import std.stdio : File;
+
+    const archive = testPath("data/zip_data-descriptor.zip");
+    const dm = DeleteMe("extraction_site", null);
+
+    mkdir(dm.path);
+
+    auto entries = File(archive, "rb")
+        .unboxZip()
+        .map!(e => tuple(e.type, e.path))
+        .array;
+
+    assert(entries.canFind(tuple(EntryType.directory, "test-zip/")));
+    assert(entries.canFind(tuple(EntryType.directory, "test-zip/test-dir/")));
+    assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test.txt")));
+    assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test-link.txt")));
+    assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test-parent.txt")));
+    assert(entries.canFind(tuple(EntryType.regular, "test-zip/test.txt")));
+    assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-link-1.txt")));
+
+    // test that reading from stream triggers an error
+    assertThrown(readBinaryFile(archive).unboxZip().array);
+}
+
 @("Extract Zip squiz-box extra-flags")
 unittest
 {

diff --git a/test/data/zip_data-descriptor.zip b/test/data/zip_data-descriptor.zip