Write meta data more compact

rrrlasse · Sep 23, 2024 · f239fc2 · f239fc2
1 parent 8ce8b53
commit f239fc2
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 44 deletions.
diff --git a/src/exdupe.cpp b/src/exdupe.cpp
@@ -302,20 +302,20 @@ STRING date2str(time_t date) {
 
 void write_contents_item(FILE *file, contents_t *c) {
     uint64_t written = io.write_count;
-    io.write_ui<uint8_t>(c->unchanged ? 1 : 0, file);
-    io.write_ui<uint32_t>(c->file_id, file);
+    uint8_t type = ((c->directory ? 1 : 0) << 0) | ((c->symlink ? 1 : 0) << 1) | ((c->unchanged ? 1 : 0) << 2);
+    io.write_ui<uint8_t>(type, file);
+    io.write_compact<uint32_t>(c->file_id, file);
+
     if(!c->unchanged) {
         io.writestr(c->abs_path, file);
-        io.write_ui<uint64_t>(c->payload, file);
+        io.write_compact<uint64_t>(c->payload, file);
         io.writestr(c->name, file);
         io.writestr(c->link, file);
-        io.write_ui<uint64_t>(c->size, file);
+        io.write_compact<uint64_t>(c->size, file);
         io.write_ui<uint32_t>(c->checksum, file);
         io.write_ui<uint32_t>(static_cast<uint32_t>(c->file_c_time), file);
         io.write_ui<uint32_t>(static_cast<uint32_t>(c->file_modified), file);
-        io.write_ui<uint32_t>(c->attributes, file);
-        io.write_ui<uint8_t>(c->directory ? 1 : 0, file);
-        io.write_ui<uint8_t>(c->symlink ? 1 : 0, file);
+        io.write_ui<uint32_t>(c->attributes, file);      
     }
     contents_size += io.write_count - written;
 }
@@ -667,22 +667,25 @@ STRING validchars(STRING filename) {
 }
 
 void read_content_item(FILE *file, contents_t *c) {
-    c->unchanged = io.read_ui<uint8_t>(file) == 0 ? false : true;
-    c->file_id = io.read_ui<uint32_t>(file);
+    uint8_t type = io.read_ui<uint8_t>(file);
+    c->directory = ((type >> 0) & 1) == 1;
+    c->symlink = ((type >> 1) & 1) == 1;
+    c->unchanged = ((type >> 2) & 1) == 1;
+
+    c->file_id = io.read_compact<uint32_t>(file);
     if(c->unchanged) {
         return;
     }
     c->abs_path = slashify(io.readstr(file));
-    c->payload = io.read_ui<uint64_t>(file);
+    c->payload = io.read_compact<uint64_t>(file);
     c->name = slashify(io.readstr(file));
     c->link = slashify(io.readstr(file));
-    c->size = io.read_ui<uint64_t>(file);
+    c->size = io.read_compact<uint64_t>(file);
     c->checksum = io.read_ui<uint32_t>(file);
     c->file_c_time = io.read_ui<uint32_t>(file);
     c->file_modified = io.read_ui<uint32_t>(file);
-    c->attributes = io.read_ui<uint32_t>(file);
-    c->directory = io.read_ui<uint8_t>(file) == 0 ? false : true;
-    c->symlink = io.read_ui<uint8_t>(file) == 0 ? false : true;
+    c->attributes = io.read_ui<uint32_t>(file);   
+
     if (!c->directory) {
         STRING i = c->name;
         c->name = slashify(validchars(c->name));

diff --git a/src/io.cpp b/src/io.cpp
@@ -126,35 +126,9 @@ size_t Cio::read_valid_length(void *DstBuf, size_t Count, FILE *_File, STRING na
     return w;
 }
 
-bool Cio::write_date(tm *t, FILE *_File) {
-    write_ui<uint8_t>(t->tm_hour, _File);
-    write_ui<uint8_t>(t->tm_isdst, _File);
-    write_ui<uint8_t>(t->tm_mday, _File);
-    write_ui<uint8_t>(t->tm_min, _File);
-    write_ui<uint8_t>(t->tm_mon, _File);
-    write_ui<uint8_t>(t->tm_sec, _File);
-    write_ui<uint8_t>(t->tm_wday, _File);
-    write_ui<uint16_t>(t->tm_yday, _File);
-    write_ui<uint16_t>(t->tm_year, _File);
-    return true;
-}
-
-bool Cio::read_date(tm *t, FILE *_File) {
-    t->tm_hour = read_ui<uint8_t>(_File);
-    t->tm_isdst = read_ui<uint8_t>(_File);
-    t->tm_mday = read_ui<uint8_t>(_File);
-    t->tm_min = read_ui<uint8_t>(_File);
-    t->tm_mon = read_ui<uint8_t>(_File);
-    t->tm_sec = read_ui<uint8_t>(_File);
-    t->tm_wday = read_ui<uint8_t>(_File);
-    t->tm_yday = read_ui<uint16_t>(_File);
-    t->tm_year = read_ui<uint16_t>(_File);
-    return true;
-}
-
 
 STRING Cio::readstr(FILE *_File) {
-    int t = read_ui<uint16_t>(_File);
+    int t = read_compact<uint16_t>(_File);
     std::string tmp = try_read(t, _File);
 #ifdef WINDOWS
     int req = MultiByteToWideChar(CP_UTF8, 0, tmp.c_str(), -1, nullptr, 0);
@@ -174,7 +148,7 @@ void Cio::writestr(STRING str, FILE *_File) {
     std::vector<char> v(req, L'c');
     WideCharToMultiByte(CP_UTF8, 0, str.c_str(), -1, &v[0], static_cast<int>(req), 0, 0);
     req--; // WideCharToMultiByte() adds trailing zero
-    write_ui<uint16_t>(static_cast<uint16_t>(req), _File); // todo, gsl::narrow
+    write_compact<uint16_t>(static_cast<uint16_t>(req), _File); // todo, gsl::narrow
     try_write(&v[0], req, _File);
 #else
     write_ui<uint16_t>(str.size(), _File);

diff --git a/src/io.hpp b/src/io.hpp
@@ -39,8 +39,6 @@ class Cio {
     FILE *open(STRING file, char mode);
     uint64_t tell(FILE *_File);
     int seek(FILE *_File, int64_t _Offset, int Origin);
-    bool write_date(struct tm *t, FILE *_File);
-    bool read_date(struct tm *t, FILE *_File);
     size_t read(void *_DstBuf, size_t _Count, FILE *_File);
     size_t write(const void *_Str, size_t _Count, FILE *_File);
     size_t try_write(const void *Str, size_t Count, FILE *_File);
@@ -75,6 +73,57 @@ class Cio {
         return value;
     }
 
+    template <typename T> requires std::is_unsigned_v<T> size_t encode_compact(T value, uint8_t* encodedBytes) {
+        size_t size = 0;
+        while (value >= 0x80) {
+            encodedBytes[size++] = static_cast<uint8_t>(value & 0x7F) | 0x80;
+            value >>= 7;
+        }
+        encodedBytes[size++] = static_cast<uint8_t>(value);
+        return size;
+    }
+
+    template <typename T> requires std::is_unsigned_v<T> T decode_compact(const uint8_t* encodedBytes) {
+        T result = 0;
+        int shift = 0;
+
+        for (uint8_t byte : encodedBytes) {
+            result |= (static_cast<T>(byte & 0x7F) << shift);
+            shift += 7;
+
+            if ((byte & 0x80) == 0) {
+                break;
+            }
+        }
+
+        return result;
+    }
+
+
+    template <typename T> requires std::is_unsigned_v<T> void write_compact(T value, FILE* f) {
+        uint8_t buf[20];
+        size_t size = encode_compact(value, buf);
+        try_write(buf, size, f);
+    }
+
+    template <typename T> requires std::is_unsigned_v<T> T read_compact(FILE* f) {
+        T result = 0;
+        int shift = 0;
+
+        for(;;) {
+            uint8_t byte;
+            try_read_buf(&byte, 1, f);
+            result |= (static_cast<T>(byte & 0x7F) << shift);
+            shift += 7;
+
+            if ((byte & 0x80) == 0) {
+                break;
+            }
+        }
+
+        return result;
+    }
+
     static bool stdin_tty();
 
 };