From fbd817add55e27f75877a75b21f72d8fdc68cff8 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Fri, 22 Sep 2023 21:55:30 +0200 Subject: [PATCH 1/5] WriteCursor.write scope param --- src/squiz_box/priv.d | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/squiz_box/priv.d b/src/squiz_box/priv.d index 52ec2e4..6b27f43 100644 --- a/src/squiz_box/priv.d +++ b/src/squiz_box/priv.d @@ -426,7 +426,7 @@ interface WriteCursor { void put(ubyte val); - void write(const(ubyte)[] arr); + void write(scope const(ubyte)[] arr); } interface SearchableWriteCursor : WriteCursor @@ -490,7 +490,7 @@ class ArrayWriteCursor : SearchableWriteCursor _writePos++; } - void write(const(ubyte)[] arr) + void write(scope const(ubyte)[] arr) { if (_writePos == _data.length) { @@ -550,7 +550,7 @@ class FileWriteCursor : WriteCursor _file.rawWrite(buf); } - void write(const(ubyte)[] arr) + void write(scope const(ubyte)[] arr) { _file.rawWrite(arr); } From 65ccefc88ae5300ff780f4b914ccb846a9e00e02 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Fri, 22 Sep 2023 21:55:49 +0200 Subject: [PATCH 2/5] WriteCursor.putValue --- src/squiz_box/priv.d | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/squiz_box/priv.d b/src/squiz_box/priv.d index 6b27f43..a1588fb 100644 --- a/src/squiz_box/priv.d +++ b/src/squiz_box/priv.d @@ -427,6 +427,13 @@ interface WriteCursor void put(ubyte val); void write(scope const(ubyte)[] arr); + + void putValue(T)(T value) + { + scope ubyte* ptr = cast(ubyte*)(&value); + scope ubyte[] arr = ptr[0 .. T.sizeof]; + write(arr); + } } interface SearchableWriteCursor : WriteCursor From 01f06b059f8519032171f914b5cb91fea6a60298 Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Fri, 22 Sep 2023 22:01:04 +0200 Subject: [PATCH 3/5] ZipUnbox support for data descriptor partial data descriptor read support that only works with seekable data source --- src/squiz_box/box/zip.d | 85 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/src/squiz_box/box/zip.d b/src/squiz_box/box/zip.d index b09c437..7f8e53f 100644 --- a/src/squiz_box/box/zip.d +++ b/src/squiz_box/box/zip.d @@ -438,6 +438,8 @@ private struct ZipUnbox(C) if (is(C : Cursor)) ulong nextHeader; Flag!"removePrefix" removePrefix; string prefix; + bool dataDescriptor; + bool zip64; static if (isSearchable) { @@ -634,15 +636,12 @@ private struct ZipUnbox(C) if (is(C : Cursor)) (flag & ZipFlag.encryption) == ZipFlag.none, "Zip encryption unsupported" ); - enforce( - (flag & ZipFlag.dataDescriptor) == ZipFlag.none, - "Zip format unsupported (data descriptor)" - ); enforce( header.compressionMethod.val == 0 || header.compressionMethod.val == 8, "Unsupported Zip compression method" ); + info.dataDescriptor = (flag & ZipFlag.dataDescriptor) != ZipFlag.none; info.deflated = header.compressionMethod.val == 8; info.expectedCrc32 = header.crc32.val; @@ -650,11 +649,13 @@ private struct ZipUnbox(C) if (is(C : Cursor)) { info.size = efInfo.uncompressedSize; info.compressedSize = efInfo.compressedSize; + info.zip64 = true; } else { info.size = header.uncompressedSize.val; info.compressedSize = header.compressedSize.val; + info.zip64 = false; } info.type = info.compressedSize == 0 ? EntryType.directory : EntryType.regular; @@ -691,6 +692,13 @@ private struct ZipUnbox(C) if (is(C : Cursor)) { import std.datetime.systime : DosFileTimeToSysTime, unixTimeToStdTime, SysTime; + if (dataDescriptor) + { + ZipDataDescriptor.read(input, zip64); + dataDescriptor = false; + zip64 = false; + } + LocalFileHeader header = void; input.readValue(&header); if (header.signature == CentralFileHeader.expectedSignature) @@ -724,6 +732,10 @@ private struct ZipUnbox(C) if (is(C : Cursor)) ZipEntryInfo info; info.path = path; fillEntryInfo(info, efInfo, header); + enforce(!info.dataDescriptor, + "ZIP files with data descriptor can only be read with seekable data source." ~ + " Try to call `unboxZip` with a `File` or `ubyte[]` input parameter." + ); // educated guess for the size in the central directory info.entrySize = header.totalLength() + info.compressedSize + @@ -757,6 +769,8 @@ private struct ZipUnbox(C) if (is(C : Cursor)) readEntry(); } + dataDescriptor = info.dataDescriptor; + zip64 = info.zip64; currentEntry = new ZipUnboxEntry!C(input, info); } } @@ -806,6 +820,8 @@ private struct ZipEntryInfo uint attributes; bool deflated; uint expectedCrc32; + bool dataDescriptor; + bool zip64; version (Posix) { @@ -814,6 +830,67 @@ private struct ZipEntryInfo } } +private struct ZipDataDescriptor +{ + enum signature = 0x08074b50; + + uint crc32; + ulong compressedSize; + ulong uncompressedSize; + + static ZipDataDescriptor read(Cursor cursor, bool zip64) + { + ZipDataDescriptor res = void; + + // the signature is optional (but what if the CRC32 actually equals the signature code ??) + auto crc32 = cursor.getValue!(LittleEndian!4)(); + if (crc32.val == signature) + { + crc32 = cursor.getValue!(LittleEndian!4)(); + } + res.crc32 = crc32.val; + + if (zip64) + { + res.compressedSize = cursor.getValue!(LittleEndian!8)().val; + res.uncompressedSize = cursor.getValue!(LittleEndian!8)().val; + } + else + { + res.compressedSize = cursor.getValue!(LittleEndian!4)().val; + res.uncompressedSize = cursor.getValue!(LittleEndian!4)().val; + } + + return res; + } + + void write(WriteCursor cursor, bool zip64) + { + assert((uncompressedSize <= 0xffffffff && compressedSize == 0xffffffff) | zip64); + + LittleEndian!4 b4 = signature; + cursor.putValue(b4); + b4 = this.crc32; + cursor.putValue(b4); + + if (zip64) + { + LittleEndian!8 b8 = this.compressedSize; + cursor.putValue(b8); + b8 = this.uncompressedSize; + cursor.putValue(b8); + } + else + { + b4 = cast(uint)this.compressedSize; + cursor.putValue(b4); + b4 = cast(uint)this.uncompressedSize; + cursor.putValue(b4); + } + } + +} + private enum KnownExtraField { none = 0, From 4cb72d75292ee7eacdc7e7b9f269d5827c96a40a Mon Sep 17 00:00:00 2001 From: Remi Thebault Date: Sun, 24 Sep 2023 03:06:24 +0200 Subject: [PATCH 4/5] add test for data-descriptor zip --- test/archive.d | 27 +++++++++++++++++++++++++++ test/data/zip_data-descriptor.zip | Bin 0 -> 1413 bytes 2 files changed, 27 insertions(+) create mode 100644 test/data/zip_data-descriptor.zip diff --git a/test/archive.d b/test/archive.d index 4d04efd..aab59f8 100644 --- a/test/archive.d +++ b/test/archive.d @@ -364,6 +364,33 @@ unittest testExtractedFiles(dm, Yes.mode666); } +@("Extract Zip With data descriptor") +unittest +{ + import std.algorithm : canFind, map; + import std.array : array; + import std.file : mkdir; + import std.stdio : File; + + const archive = testPath("data/zip_data-descriptor.zip"); + const dm = DeleteMe("extraction_site", null); + + mkdir(dm.path); + + auto entries = File(archive, "rb") + .unboxZip() + .map!(e => tuple(e.type, e.path)) + .array; + + assert(entries.canFind(tuple(EntryType.directory, "test-zip/"))); + assert(entries.canFind(tuple(EntryType.directory, "test-zip/test-dir/"))); + assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test.txt"))); + assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test-link.txt"))); + assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test-parent.txt"))); + assert(entries.canFind(tuple(EntryType.regular, "test-zip/test.txt"))); + assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-link-1.txt"))); +} + @("Extract Zip squiz-box extra-flags") unittest { diff --git a/test/data/zip_data-descriptor.zip b/test/data/zip_data-descriptor.zip new file mode 100644 index 0000000000000000000000000000000000000000..e2cc2e694bd0e4ed80183e5ab80bac7bd2670d43 GIT binary patch literal 1413 zcma))Jxjw-6oyYfw9&RorGvPrqeB|}142P)OF&cn0w;xbD3wy%2vu?DKM))orIVA0 zSafi4aA`sM3o-~D92MtM@44-{iI)U(!^sVSJkLERuc3#+Nx*gYSl;CiRG9(?QF*(a@&s;s7@MD2!uaF0Q_bM%(H}+ zLvYG$nEad>#-)Qa!8{D|U-Gwzu*4Buy%%{WakfFC9~yW~PK} z5)B+PDwRZz=G0VP>Grgb@#Ho)9%2lJK)Wf-(aqJ$cY Date: Sun, 24 Sep 2023 03:12:53 +0200 Subject: [PATCH 5/5] test that reading from stream triggers an error --- test/archive.d | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/archive.d b/test/archive.d index aab59f8..73cc4c7 100644 --- a/test/archive.d +++ b/test/archive.d @@ -369,6 +369,7 @@ unittest { import std.algorithm : canFind, map; import std.array : array; + import std.exception : assertThrown; import std.file : mkdir; import std.stdio : File; @@ -389,6 +390,9 @@ unittest assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-dir/test-parent.txt"))); assert(entries.canFind(tuple(EntryType.regular, "test-zip/test.txt"))); assert(entries.canFind(tuple(EntryType.regular, "test-zip/test-link-1.txt"))); + + // test that reading from stream triggers an error + assertThrown(readBinaryFile(archive).unboxZip().array); } @("Extract Zip squiz-box extra-flags")