support Interface hsync_TDictCompress new API getDictCompressBorder();

(This new api for optimize libdeflate dict compress speed to 10x) close #30
sisong · Jul 5, 2024 · b68b046 · b68b046
1 parent df19b22
commit b68b046
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 63 deletions.
diff --git a/HDiffPatch b/HDiffPatch
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 # [hsynz](https://github.com/sisong/hsynz)
-[![release](https://img.shields.io/badge/release-v1.1.0-blue.svg)](https://github.com/sisong/hsynz/releases) 
+[![release](https://img.shields.io/badge/release-v1.1.1-blue.svg)](https://github.com/sisong/hsynz/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/hsynz/blob/main/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/hsynz/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/hsynz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/hsynz/issues)   
@@ -83,11 +83,9 @@ options:
         -c-gzip[-{1..9}[-dictBits]]     DEFAULT level 9
             dictBits can 9--15, DEFAULT 15.
             compress by zlib, out_hsynz_file is .gz file format.
-        -c-ldef[-{1..12}[-dictBits]]    DEFAULT level 12
-            dictBits can 9--15, DEFAULT 15.
+        -c-ldef[-{1..12}]           DEFAULT level 12 (dictBits always 15).
             compress by libdeflate, compatible with zlib's deflate encoding.
-        -c-lgzip[-{1..12}[-dictBits]]   DEFAULT level 12
-            dictBits can 9--15, DEFAULT 15.
+        -c-lgzip[-{1..12}]          DEFAULT level 12 (dictBits always 15)
             compress by libdeflate, out_hsynz_file is .gz file format.
         -c-zstd[-{10..22}[-dictBits]]   DEFAULT level 21
             dictBits can 15--30, DEFAULT 24.
@@ -206,7 +204,7 @@ case list([download from OneDrive](https://1drv.ms/u/s!Aj8ygMPeifoQgUIZxYac5_ufl
 
 
 **test PC**: Windows11, CPU R9-7945HX, SSD PCIe4.0x4 4T, DDR5 5200MHz 32Gx2   
-**Program version**: hsynz 1.1.0, zsync 0.6.2  (more programs's testing see [HDiffPatch](https://github.com/sisong/HDiffPatch))   
+**Program version**: hsynz 1.1.1, zsync 0.6.2  (more programs's testing see [HDiffPatch](https://github.com/sisong/HDiffPatch))   
 **test Program**:   
 **zsync** run make with `zsyncmake -b 2048 -o {out_newi} {new}`,   
 client sync diff&patch by `zsync -i {old} -o {out_new} {newi}` (all files are local)   
@@ -237,10 +235,10 @@ client sync diff&patch by `hsync_demo {old} {newi} {newz} {out_new}` (all files
 |hsynz p8 zlib|20.05%|30M|115.1MB/s|13M|29M|435MB/s|
 |hsynz p1 gzip|20.12%|6M|17.3MB/s|6M|22M|268MB/s|
 |hsynz p8 gzip|20.12%|30M|115.0MB/s|13M|29M|427MB/s|
-|hsynz p1 ldef|19.58%|15M|1.0MB/s|6M|22M|272MB/s|
-|hsynz p8 ldef|19.58%|96M|7.2MB/s|13M|29M|431MB/s|
-|hsynz p1 lgzip|19.66%|15M|1.0MB/s|6M|22M|267MB/s|
-|hsynz p8 lgzip|19.66%|96M|7.2MB/s|13M|29M|419MB/s|
+|hsynz p1 ldef|19.57%|15M|7.8MB/s|6M|22M|272MB/s|
+|hsynz p8 ldef|19.57%|96M|57.0MB/s|13M|29M|431MB/s|
+|hsynz p1 lgzip|19.64%|15M|7.9MB/s|6M|22M|267MB/s|
+|hsynz p8 lgzip|19.64%|96M|56.9MB/s|13M|29M|419MB/s|
 |hsynz p1 zstd|14.96%|532M|1.9MB/s|24M|34M|264MB/s|
 |hsynz p8 zstd|14.95%|3349M|10.1MB/s|24M|34M|410MB/s|
 
@@ -298,10 +296,10 @@ case list:
 |hsynz p8 zlib|58.67%|29M|138.6MB/s|12M|19M|410MB/s|
 |hsynz p1 gzip|58.95%|5M|22.6MB/s|4M|11M|242MB/s|
 |hsynz p8 gzip|58.95%|29M|138.9MB/s|12M|19M|407MB/s|
-|hsynz p1 ldef|58.62%|14M|1.9MB/s|4M|11M|242MB/s|
-|hsynz p8 ldef|58.62%|96M|11.3MB/s|12M|19M|413MB/s|
-|hsynz p1 lgzip|58.90%|14M|1.9MB/s|4M|11M|240MB/s|
-|hsynz p8 lgzip|58.90%|96M|11.3MB/s|12M|19M|405MB/s|
+|hsynz p1 ldef|58.61%|14M|23.7MB/s|4M|11M|242MB/s|
+|hsynz p8 ldef|58.61%|96M|149.1MB/s|12M|19M|413MB/s|
+|hsynz p1 lgzip|58.90%|14M|23.6MB/s|4M|11M|240MB/s|
+|hsynz p8 lgzip|58.90%|96M|149.1MB/s|12M|19M|405MB/s|
 |hsynz p1 zstd|57.74%|534M|2.7MB/s|24M|28M|234MB/s|
 |hsynz p8 zstd|57.74%|3434M|13.4MB/s|24M|28M|390MB/s|
 

diff --git a/README_cn.md b/README_cn.md
@@ -1,5 +1,5 @@
 # [hsynz](https://github.com/sisong/hsynz)
-[![release](https://img.shields.io/badge/release-v1.1.0-blue.svg)](https://github.com/sisong/hsynz/releases) 
+[![release](https://img.shields.io/badge/release-v1.1.1-blue.svg)](https://github.com/sisong/hsynz/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/hsynz/blob/main/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/hsynz/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/hsynz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/hsynz/issues)   
@@ -85,11 +85,11 @@ hsync_make: [options] newDataPath out_hsyni_file [out_hsynz_file]
             压缩字典比特数dictBits可以为9到15, 默认为15。
             使用zlib算法来压缩, out_hsynz_file输出文件将是一个标准的.gz格式文件。
             （会比 -c-zlib 生成的文件稍大一点）
-        -c-ldef[-{1..12}[-dictBits]]    默认级别 12
-            压缩字典比特数dictBits可以为9到15, 默认为15。
+        -c-ldef[-{1..12}]    默认级别 12
+            压缩字典比特数dictBits总是设置为15。
             使用libdeflate算法来压缩, 兼容zlib的deflate编码。
-        -c-lgzip[-{1..12}[-dictBits]]   默认级别 12
-            压缩字典比特数dictBits可以为9到15, 默认为15。
+        -c-lgzip[-{1..12}]   默认级别 12
+            压缩字典比特数dictBits总是设置为15。
             使用libdeflate算法来压缩, out_hsynz_file输出文件将是一个标准的.gz格式文件。
         -c-zstd[-{10..22}[-dictBits]]   默认级别 21
             压缩字典比特数dictBits 可以为15到30, 默认为24。
@@ -202,7 +202,7 @@ hsync_make: [options] newDataPath out_hsyni_file [out_hsynz_file]
 
 
 **测试PC**: Windows11, CPU R9-7945HX, SSD PCIe4.0x4 4T, DDR5 5200MHz 32Gx2   
-**参与测试的程序版本**: hsynz 1.1.0, zsync 0.6.2  (更多程序的对比测试结果见 [HDiffPatch](https://github.com/sisong/HDiffPatch))   
+**参与测试的程序版本**: hsynz 1.1.1, zsync 0.6.2  (更多程序的对比测试结果见 [HDiffPatch](https://github.com/sisong/HDiffPatch))   
 **程序测试参数**:   
 **zsync** 运行 make 参数 `zsyncmake -b 2048 -o {out_newi} {new}`,   
 客户端同步 diff&patch 时参数 `zsync -i {old} -o {out_new} {newi}` (所有文件都在本地)   
@@ -233,10 +233,10 @@ hsync_make: [options] newDataPath out_hsyni_file [out_hsynz_file]
 |hsynz p8 zlib|20.05%|30M|115.1MB/s|13M|29M|435MB/s|
 |hsynz p1 gzip|20.12%|6M|17.3MB/s|6M|22M|268MB/s|
 |hsynz p8 gzip|20.12%|30M|115.0MB/s|13M|29M|427MB/s|
-|hsynz p1 ldef|19.58%|15M|1.0MB/s|6M|22M|272MB/s|
-|hsynz p8 ldef|19.58%|96M|7.2MB/s|13M|29M|431MB/s|
-|hsynz p1 lgzip|19.66%|15M|1.0MB/s|6M|22M|267MB/s|
-|hsynz p8 lgzip|19.66%|96M|7.2MB/s|13M|29M|419MB/s|
+|hsynz p1 ldef|19.57%|15M|7.8MB/s|6M|22M|272MB/s|
+|hsynz p8 ldef|19.57%|96M|57.0MB/s|13M|29M|431MB/s|
+|hsynz p1 lgzip|19.64%|15M|7.9MB/s|6M|22M|267MB/s|
+|hsynz p8 lgzip|19.64%|96M|56.9MB/s|13M|29M|419MB/s|
 |hsynz p1 zstd|14.96%|532M|1.9MB/s|24M|34M|264MB/s|
 |hsynz p8 zstd|14.95%|3349M|10.1MB/s|24M|34M|410MB/s|
 
@@ -294,10 +294,10 @@ case list:
 |hsynz p8 zlib|58.67%|29M|138.6MB/s|12M|19M|410MB/s|
 |hsynz p1 gzip|58.95%|5M|22.6MB/s|4M|11M|242MB/s|
 |hsynz p8 gzip|58.95%|29M|138.9MB/s|12M|19M|407MB/s|
-|hsynz p1 ldef|58.62%|14M|1.9MB/s|4M|11M|242MB/s|
-|hsynz p8 ldef|58.62%|96M|11.3MB/s|12M|19M|413MB/s|
-|hsynz p1 lgzip|58.90%|14M|1.9MB/s|4M|11M|240MB/s|
-|hsynz p8 lgzip|58.90%|96M|11.3MB/s|12M|19M|405MB/s|
+|hsynz p1 ldef|58.61%|14M|23.7MB/s|4M|11M|242MB/s|
+|hsynz p8 ldef|58.61%|96M|149.1MB/s|12M|19M|413MB/s|
+|hsynz p1 lgzip|58.90%|14M|23.6MB/s|4M|11M|240MB/s|
+|hsynz p8 lgzip|58.90%|96M|149.1MB/s|12M|19M|405MB/s|
 |hsynz p1 zstd|57.74%|534M|2.7MB/s|24M|28M|234MB/s|
 |hsynz p8 zstd|57.74%|3434M|13.4MB/s|24M|28M|390MB/s|
 

diff --git a/dict_compress_plugin_demo.h b/dict_compress_plugin_demo.h
@@ -157,9 +157,8 @@ static const char* k_gzip_dictCompressType="gzipD";
 
     static size_t _zlib_dictCompress(hsync_dictCompressHandle dictHandle,size_t blockIndex,
                                      hpatch_byte* out_code,hpatch_byte* out_codeEnd,
-                                     const hpatch_byte* in_dataBegin,const hpatch_byte* in_dataEnd){
+                                     const hpatch_byte* in_dataBegin,size_t in_dataSize,size_t in_borderSize){
         _TDictCompressPlugin_zlib_data* self=(_TDictCompressPlugin_zlib_data*)dictHandle;
-        const size_t dataSize=in_dataEnd-in_dataBegin;
         const hpatch_BOOL in_isEnd=(blockIndex+1==self->cache.blockCount);
         size_t result;
         if (self->dict_isInReset){ //reset dict
@@ -172,8 +171,8 @@ static const char* k_gzip_dictCompressType="gzipD";
             self->dict_isInReset=hpatch_FALSE;
         }
         self->stream.next_in =(Bytef*)in_dataBegin;
-        self->stream.avail_in=(uInt)dataSize;
-        assert(self->stream.avail_in==dataSize);
+        self->stream.avail_in=(uInt)in_dataSize;
+        assert(self->stream.avail_in==in_dataSize);
         self->stream.next_out =out_code;
         self->stream.avail_out=(uInt)(out_codeEnd-out_code);
         assert(self->stream.avail_out==(size_t)(out_codeEnd-out_code));
@@ -200,15 +199,15 @@ static const char* k_gzip_dictCompressType="gzipD";
                 #else
                     //update dict & deflateSetDictionary every block (NOTE: deflateSetDictionary slow)
                     _CacheBlockDict_dictUncompress(&self->cache,blockIndex,blockIndex+1,
-                                                   in_dataBegin,in_dataEnd);
+                                                   in_dataBegin,in_dataBegin+in_dataSize);
                     self->dict_isInReset=hpatch_TRUE;
                 #endif
             }
         }
         result=self->stream.total_out;
         self->stream.total_out=0;
         assert(self->stream.avail_in==0);
-        if ((result>=dataSize)&&(!self->is_gzip))
+        if ((result>=in_dataSize)&&(!self->is_gzip))
             return kDictCompressCancel;// cancel compress
         return result;
     }
@@ -250,9 +249,13 @@ static const char* k_gzip_dictCompressType="gzipD";
         struct libdeflate_compressor* c;
         _CacheBlockDict_t   cache;
         hpatch_byte*        tempDecBufEnd; 
+        hpatch_BOOL         dict_isInReset;
         hpatch_BOOL         is_gzip;
     } _TDictCompressPlugin_ldef_data;
 
+    static size_t _ldef_getDictCompressBorder(){
+        return 258; //deflate max match length
+    }
     static size_t _ldef_getDictSize(struct hsync_TDictCompress* compressPlugin){
         TDictCompressPlugin_ldef*  plugin=(TDictCompressPlugin_ldef*)compressPlugin;
         const size_t dictSize=((size_t)1<<plugin->dict_bits);
@@ -271,10 +274,11 @@ static const char* k_gzip_dictCompressType="gzipD";
         const TDictCompressPlugin_ldef*  plugin=(const TDictCompressPlugin_ldef*)compressPlugin;
         const size_t dictSize=((size_t)1<<plugin->dict_bits);
         const size_t cacheDictSize=_getCacheBlockDictSize(dictSize,blockCount,blockSize);
-        _TDictCompressPlugin_ldef_data* self=(_TDictCompressPlugin_ldef_data*)malloc(sizeof(_TDictCompressPlugin_ldef_data)+cacheDictSize+blockSize);
+        const size_t memSize=sizeof(_TDictCompressPlugin_ldef_data)+cacheDictSize+blockSize+_ldef_getDictCompressBorder();
+        _TDictCompressPlugin_ldef_data* self=(_TDictCompressPlugin_ldef_data*)malloc(memSize);
         if (self==0) return 0;
         memset(self,0,sizeof(*self));
-        self->tempDecBufEnd=((hpatch_byte*)self)+sizeof(*self) +cacheDictSize+blockSize;
+        self->tempDecBufEnd=((hpatch_byte*)self)+memSize;
         _CacheBlockDict_init(&self->cache,((hpatch_byte*)self)+sizeof(*self),
                              cacheDictSize,dictSize,blockCount,blockSize);
         self->c=libdeflate_alloc_compressor(plugin->compress_level);
@@ -297,37 +301,43 @@ static const char* k_gzip_dictCompressType="gzipD";
     static hpatch_byte* _ldef_getResetDictBuffer(hsync_dictCompressHandle dictHandle,size_t blockIndex,
                                                  size_t* out_dictSize){
         _TDictCompressPlugin_ldef_data* self=(_TDictCompressPlugin_ldef_data*)dictHandle;
+        assert(!self->dict_isInReset);
+        self->dict_isInReset=hpatch_TRUE;
         return _CacheBlockDict_getResetDictBuffer(&self->cache,blockIndex,out_dictSize);
     }
 
     static size_t _ldef_dictCompress(hsync_dictCompressHandle dictHandle,size_t blockIndex,
                                      hpatch_byte* out_code,hpatch_byte* out_codeEnd,
-                                     const hpatch_byte* in_dataBegin,const hpatch_byte* in_dataEnd){
+                                     const hpatch_byte* in_dataBegin,size_t in_dataSize,size_t in_borderSize){
         _TDictCompressPlugin_ldef_data* self=(_TDictCompressPlugin_ldef_data*)dictHandle;
-        const size_t dataSize=in_dataEnd-in_dataBegin;
         const hpatch_BOOL in_isEnd=(blockIndex+1==self->cache.blockCount);
         size_t result;
         hpatch_byte* dict;
         size_t       dictSize;
         hpatch_BOOL  isHaveDict;
         const hpatch_BOOL  isSetFinalTag=in_isEnd||(!self->is_gzip);
-        { //reset dict
+        { //reset dict?
             _CacheBlockDict_usedDict(&self->cache,blockIndex,&dict,&dictSize);
             isHaveDict=(dictSize>0);
             if (isHaveDict){
-                _checkCompress(dataSize<=(size_t)(self->tempDecBufEnd-dict-dictSize));
-                memcpy(dict+dictSize,in_dataBegin,dataSize);
+                _checkCompress(in_dataSize+in_borderSize<=(size_t)(self->tempDecBufEnd-dict-dictSize));
+                memcpy(dict+dictSize,in_dataBegin,in_dataSize+in_borderSize);
+            }
+            if (self->dict_isInReset){
+                self->dict_isInReset=hpatch_FALSE;
+                libdeflate_deflate_compress_block_reset(self->c);
             }
         }
 
-        result=libdeflate_deflate_compress_block(self->c,
-                    isHaveDict?dict:in_dataBegin,dictSize,dataSize,isSetFinalTag,
-                    out_code,out_codeEnd-out_code,1);
+        //result=libdeflate_deflate_compress_block(self->c, isHaveDict?dict:in_dataBegin,
+        //            dictSize,in_dataSize,isSetFinalTag, out_code,out_codeEnd-out_code,1);
+        result=libdeflate_deflate_compress_block_continue(self->c, isHaveDict?dict:in_dataBegin,
+                    dictSize,in_dataSize,in_borderSize,isSetFinalTag, out_code,out_codeEnd-out_code,1);
         _checkCompress(result>0);
         if (!in_isEnd)
             _CacheBlockDict_dictUncompress(&self->cache,blockIndex,blockIndex+1,
-                                           in_dataBegin,in_dataEnd);
-        if ((result>=dataSize)&&(!self->is_gzip))
+                                           in_dataBegin,in_dataBegin+in_dataSize);
+        if ((result>=in_dataSize)&&(!self->is_gzip))
             return kDictCompressCancel;// cancel compress
         return result;
     }
@@ -339,7 +349,8 @@ static const char* k_gzip_dictCompressType="gzipD";
         {_ldef_dictCompressType,_default_maxCompressedSize,
             _ldef_limitDictSizeByData,_default_getBestWorkBlockCount,
             _ldef_getDictSize,_ldef_dictCompressOpen,_ldef_dictCompressClose,0,
-            _ldef_getResetDictBuffer,_ldef_dictCompress,_ldef_dictCompressTypeForDisplay},
+            _ldef_getResetDictBuffer,_ldef_dictCompress,
+            _ldef_dictCompressTypeForDisplay,_ldef_getDictCompressBorder},
         12,MAX_WBITS,hpatch_FALSE};
     _def_fun_compressType(_lgzip_dictCompressType,k_gzip_dictCompressType);
     _def_fun_compressType(_lgzip_dictCompressTypeForDisplay,"lgzipD (gzipD compatible)");
@@ -348,7 +359,8 @@ static const char* k_gzip_dictCompressType="gzipD";
         {_lgzip_dictCompressType,_default_maxCompressedSize,
             _ldef_limitDictSizeByData,_default_getBestWorkBlockCount,
             _ldef_getDictSize,_ldef_dictCompressOpen,_ldef_dictCompressClose,0,
-            _ldef_getResetDictBuffer,_ldef_dictCompress,_lgzip_dictCompressTypeForDisplay},
+            _ldef_getResetDictBuffer,_ldef_dictCompress,
+            _lgzip_dictCompressTypeForDisplay,_ldef_getDictCompressBorder},
         12,MAX_WBITS,hpatch_TRUE};
 
 #endif//_CompressPlugin_ldef
@@ -502,12 +514,11 @@ static const char* k_gzip_dictCompressType="gzipD";
 
     static size_t _zstd_dictCompress(hsync_dictCompressHandle dictHandle,size_t blockIndex,
                                      hpatch_byte* out_code,hpatch_byte* out_codeEnd,
-                                     const hpatch_byte* in_dataBegin,const hpatch_byte* in_dataEnd){
+                                     const hpatch_byte* in_dataBegin,size_t in_dataSize,size_t in_borderSize){
         _TDictCompressPlugin_zstd_data* self=(_TDictCompressPlugin_zstd_data*)dictHandle;
         ZSTD_CCtx* s=self->s;
         ZSTD_inBuffer       s_input;
         ZSTD_outBuffer      s_output;
-        const size_t dataSize=in_dataEnd-in_dataBegin;
         const hpatch_BOOL in_isEnd=blockIndex+1==self->cache.blockCount;
         if (_CacheBlockDict_isHaveDict(&self->cache)){ //reset dict
             _zstd_checkComp(ZSTD_CCtx_reset(s,ZSTD_reset_session_only));
@@ -525,7 +536,7 @@ static const char* k_gzip_dictCompressType="gzipD";
         }
 
         s_input.src=in_dataBegin;
-        s_input.size=dataSize;
+        s_input.size=in_dataSize;
         s_input.pos=0;
         s_output.dst=out_code;
         s_output.size=out_codeEnd-out_code;
@@ -534,10 +545,10 @@ static const char* k_gzip_dictCompressType="gzipD";
         assert(s_input.pos==s_input.size);
         if (!in_isEnd){//update cache & dict
             const hpatch_byte* lastUpdated=self->cache.uncompressCur;
-            _CacheBlockDict_dictUncompress(&self->cache,blockIndex,blockIndex+1,in_dataBegin,in_dataEnd);
+            _CacheBlockDict_dictUncompress(&self->cache,blockIndex,blockIndex+1,in_dataBegin,in_dataBegin+in_dataSize);
             if (self->isDeltaDict){
-                if ((self->d!=0)&&(lastUpdated+dataSize==self->cache.uncompressCur)){
-                    _zstd_checkComp(ZSTD_updateCDict_delta(self->d,dataSize));
+                if ((self->d!=0)&&(lastUpdated+in_dataSize==self->cache.uncompressCur)){
+                    _zstd_checkComp(ZSTD_updateCDict_delta(self->d,in_dataSize));
                 }else{
                     __zstd_reCreateCDict(self);
                     _checkCompress(self->d!=0);

diff --git a/hsync_import_patch.h b/hsync_import_patch.h
@@ -13,7 +13,7 @@ extern "C" {
 #endif
 #define HSYNC_VERSION_MAJOR    1
 #define HSYNC_VERSION_MINOR    1
-#define HSYNC_VERSION_RELEASE  0
+#define HSYNC_VERSION_RELEASE  1
 
 #define _HSYNC_VERSION          HSYNC_VERSION_MAJOR.HSYNC_VERSION_MINOR.HSYNC_VERSION_RELEASE
 #define _HSYNC_QUOTE(str) #str
+2 −1		libhsync/sync_make/dict_compress_plugin.h
+1 −1		libhsync/sync_make/sync_info_make.cpp
+24 −14		libhsync/sync_make/sync_make.cpp