diff --git a/CHANGELOG.md b/CHANGELOG.md index c5732496..7bcdce82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ full changelog at: https://github.com/sisong/HDiffPatch/commits +## [v5.1.0](https://github.com/sisong/HDiffPatch/tree/v5.1.0) - 2026-07-14 +### Added +* optimize `$hdiffz -m` required memory size when oldData different newData; + default opened by diff with -block or -cache opened; +* add lzma2mtDecompressPlugin, support lzma2 decompress by multi-thread; + ## [v5.0.0](https://github.com/sisong/HDiffPatch/tree/v5.0.0) - 2026-06-25 ### Added * add new format Window Diff(`HDIFFW26`) for optimize patch speed, by `$hdiffz -WD[-stepSize]`; diff --git a/Makefile b/Makefile index 276c8594..e198b55b 100644 --- a/Makefile +++ b/Makefile @@ -119,7 +119,8 @@ ifeq ($(LZMA),1) ifeq ($(MT),0) else HPATCH_OBJ+=$(LZMA_PATH)/MtDec.o \ - $(LZMA_PATH)/Threads.o + $(LZMA_PATH)/Threads.o \ + $(LZMA_PATH)/Lzma2DecMt.o endif HDIFF_OBJ += $(LZMA_PATH)/LzFind.o \ $(LZMA_PATH)/LzFindOpt.o \ diff --git a/README.md b/README.md index 5ba7180f..ef8afa0f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [HDiffPatch] -[![release](https://img.shields.io/badge/release-v5.0.1-blue.svg)](https://github.com/sisong/HDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v5.1.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues) diff --git a/README_cn.md b/README_cn.md index 2aaa0df9..88b409f0 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # [HDiffPatch] -[![release](https://img.shields.io/badge/release-v5.0.1-blue.svg)](https://github.com/sisong/HDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v5.1.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues) diff --git a/bsdiff_wrapper/bsdiff_wrapper.cpp b/bsdiff_wrapper/bsdiff_wrapper.cpp index b2fca803..f4f3c865 100644 --- a/bsdiff_wrapper/bsdiff_wrapper.cpp +++ b/bsdiff_wrapper/bsdiff_wrapper.cpp @@ -57,7 +57,7 @@ namespace hdiff_private{ :curPos(0),curi(0),bufi(0),covers(_covers){ streamImport=this; read=_read; - streamSize=(covers.size()-1)*(hpatch_StreamPos_t)(3*8); + streamSize=(covers.size()>1)?((hpatch_StreamPos_t)covers.size()-1)*(3*8):0; buf.reserve(hdiff_kFileIOBufBestSize); } private: diff --git a/builds/android_ndk_jni_mk/hpatch.c b/builds/android_ndk_jni_mk/hpatch.c index 0ee11373..25220a29 100644 --- a/builds/android_ndk_jni_mk/hpatch.c +++ b/builds/android_ndk_jni_mk/hpatch.c @@ -37,15 +37,16 @@ void bz_internal_error(int errcode){ int hpatchz(const char *oldFileName,const char *diffFileName,const char *outNewFileName, int64_t cacheMemory,size_t threadNum,hpatch_BOOL isChecksumNewData){ + size_t dec_threadNum=1;//now only support lzma2 decompressor,& used large memory! TPatchChecksumSet checksumSet={0,hpatch_FALSE,isChecksumNewData,isChecksumNewData,hpatch_FALSE}; #if (_IS_NEED_DIR_DIFF_PATCH) const hpatch_BOOL isDirDiff=getIsDirDiffFile(diffFileName); if (isDirDiff){ return hpatch_dir(oldFileName,diffFileName,outNewFileName, hpatch_FALSE,limitCacheMemory(cacheMemory),kMaxOpenFileNumber_default_patch, - &checksumSet,&defaultPatchDirlistener,0,0,threadNum); + &checksumSet,&defaultPatchDirlistener,0,0,threadNum,dec_threadNum); } else #endif return hpatch(oldFileName,diffFileName,outNewFileName, - hpatch_FALSE,limitCacheMemory(cacheMemory),0,0,&checksumSet,threadNum); + hpatch_FALSE,limitCacheMemory(cacheMemory),0,0,&checksumSet,threadNum,dec_threadNum); } diff --git a/compress_plugin_demo.h b/compress_plugin_demo.h index e0baf391..bdf05e91 100644 --- a/compress_plugin_demo.h +++ b/compress_plugin_demo.h @@ -812,11 +812,15 @@ int _default_setParallelThreadNumber(hdiff_TCompress* compressPlugin,int threadN #ifdef _CompressPlugin_lzma2 #if (_IsNeedIncludeDefaultCompressHead) # include "MtCoder.h" // "lzma/C/MtCoder.h" for MTCODER__THREADS_MAX +#endif +#ifndef LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT +# define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1<<28) // 256M #endif struct TCompressPlugin_lzma2{ hdiff_TCompress base; int compress_level; //0..9 UInt32 dict_size; //patch decompress need 4?*lzma_dictSize memory + UInt32 block_size; //1M--256M; 0=AUTO, LZMA2 chunk size, affects decompress parallelism int thread_num; //1..(64?) }; static int _lzma2_setThreadNumber(hdiff_TCompress* compressPlugin,int threadNum){ @@ -848,7 +852,8 @@ int _default_setParallelThreadNumber(hdiff_TCompress* compressPlugin,int threadN props.lzmaProps.level=plugin->compress_level; props.lzmaProps.dictSize=dictSize; props.lzmaProps.reduceSize=in_data->streamSize; - props.blockSize=LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO; + props.blockSize=(plugin->block_size==0)?LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO: + (plugin->block_sizeblock_size:LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; props.numTotalThreads=plugin->thread_num; Lzma2EncProps_Normalize(&props); if (SZ_OK!=Lzma2Enc_SetProps(s,&props)) _compress_error_return("Lzma2Enc_SetProps()"); @@ -885,7 +890,7 @@ int _default_setParallelThreadNumber(hdiff_TCompress* compressPlugin,int threadN _def_fun_compressType(_lzma2_compressType,"lzma2"); static const TCompressPlugin_lzma2 lzma2CompressPlugin={ {_lzma2_compressType,_default_maxCompressedSize,_lzma2_setThreadNumber,_lzma2_compress}, - 7,(1<<23),kDefaultCompressThreadNumber}; + 7,(1<<23),0,kDefaultCompressThreadNumber}; #endif//_CompressPlugin_lzma2 #ifdef _CompressPlugin_7zXZ diff --git a/decompress_plugin_demo.h b/decompress_plugin_demo.h index db5070d3..b45f2146 100644 --- a/decompress_plugin_demo.h +++ b/decompress_plugin_demo.h @@ -33,13 +33,15 @@ // bz2DecompressPlugin; // lzmaDecompressPlugin; // lzma2DecompressPlugin; +// lzma2mtDecompressPlugin; // lz4DecompressPlugin; // zstdDecompressPlugin; // brotliDecompressPlugin; // lzhamDecompressPlugin; // tuzDecompressPlugin; -// _bz2DecompressPlugin_unsz : support for bspatch_with_cache() +// _bz2DecompressPlugin_unsz : support for bspatch_with_cache(), diffData created by bsdiff or "hdiffz -BSD ..." +// _lzma2DecompressPlugin_unsz : only for yourself code, support for bspatch_with_cache(), diffData compressed by lzma2 (lzma2CompressPlugin) // _7zXZDecompressPlugin : support for vcpatch_with_cache(), diffData created by "xdelta3 -S lzma ..." // _7zXZDecompressPlugin_a : support for vcpatch_with_cache(), diffData created by "hdiffz -VCD-compressLevel ..." #include //malloc free @@ -711,6 +713,10 @@ static void __dec_free(void* _, void* address){ # include "LzmaDec.h" // "lzma/C/LzmaDec.h" https://github.com/sisong/lzma # ifdef _CompressPlugin_lzma2 # include "Lzma2Dec.h" +# if defined(_IS_USED_MULTITHREAD) && !defined(Z7_ST) +# include "Lzma2DecMt.h" +# include "libParallel/parallel_import_c.h" +# endif # endif #endif #endif @@ -889,8 +895,9 @@ static void __dec_free(void* _, void* address){ free(self); return hpatch_TRUE; } - static hpatch_BOOL _lzma2_decompress_part(hpatch_decompressHandle decompressHandle, - unsigned char* out_part_data,unsigned char* out_part_data_end){ + static hpatch_BOOL _lzma2_decompress_part_(hpatch_decompressHandle decompressHandle, + unsigned char* out_part_data,unsigned char* out_part_data_end, + hpatch_BOOL isMustOutData){ _lzma2_TDecompress* self=(_lzma2_TDecompress*)decompressHandle; unsigned char* out_cur=out_part_data; assert(out_part_data<=out_part_data_end); @@ -926,8 +933,14 @@ static void __dec_free(void* _, void* address){ res=Lzma2Dec_DecodeToDic(&self->decEnv,self->decEnv.decoder.dicBufSize, self->dec_buf+self->decReadPos,&inSize,LZMA_FINISH_ANY,&status); if(res==SZ_OK){ - if ((inSize==0)&&(self->decEnv.decoder.dicPos==dicPos_back)) - _dec_onDecErr_rt();//error; + if ((inSize==0)&&(self->decEnv.decoder.dicPos==dicPos_back)){ + if (isMustOutData){ //fill out 0 + memset(out_cur,0,out_part_data_end-out_cur); + return hpatch_TRUE; + }else{ + _dec_onDecErr_rt();//error; + } + } }else{ _dec_onDecErr_rt();//error; } @@ -936,10 +949,276 @@ static void __dec_free(void* _, void* address){ } return hpatch_TRUE; } + + static hpatch_BOOL _lzma2_decompress_part(hpatch_decompressHandle decompressHandle, + unsigned char* out_part_data,unsigned char* out_part_data_end){ + return _lzma2_decompress_part_(decompressHandle,out_part_data,out_part_data_end,hpatch_FALSE); + } + static hpatch_BOOL _lzma2_decompress_part_unsz(hpatch_decompressHandle decompressHandle, + unsigned char* out_part_data,unsigned char* out_part_data_end){ + return _lzma2_decompress_part_(decompressHandle,out_part_data,out_part_data_end,hpatch_TRUE); + } + + + static hpatch_TDecompress lzma2DecompressPlugin={_lzma2_is_can_open,_lzma2_open, _lzma2_close,_lzma2_decompress_part}; + //unkown uncompress data size + static hpatch_TDecompress _lzma2DecompressPlugin_unsz={_lzma2_is_can_open,_lzma2_open, + _lzma2_close,_lzma2_decompress_part_unsz}; #endif//_CompressPlugin_lzma2 + +//--- lzma2 multi-thread decompress --- +#if defined(_CompressPlugin_lzma2) && defined(_IS_USED_MULTITHREAD) && (!defined(Z7_ST)) +#ifndef _CompressPlugin_lzma2mt +# define _CompressPlugin_lzma2mt 1 +#endif +#endif + +#if (_CompressPlugin_lzma2mt) + +#define kLzma2mtRingBufSize (1<<22) // 4MB + +typedef struct { + unsigned char* buf; + size_t capacity; + size_t r, w, used; + HLocker locker; + HCondvar hasSpaceCond; + HCondvar hasDataCond; +} _lzma2mt_ringbuf; + +typedef struct { + ISeqInStream vt; + const struct hpatch_TStreamInput* codeStream; + hpatch_StreamPos_t curPos, endPos; + hpatch_BOOL volatile* pIsClosing; +} _lzma2mt_InStream; + +typedef struct { + ISeqOutStream vt; + struct _lzma2mt_TDecompress* dec; +} _lzma2mt_OutStream; + +typedef struct _lzma2mt_TDecompress { + ISzAlloc memAllocBase; + _lzma2mt_ringbuf ring; + _lzma2mt_InStream inStream; + _lzma2mt_OutStream outStream; + CLzma2DecMtHandle decMtHandle; + CLzma2DecMtProps mtProps; + Byte propByte; + UInt64 dataSize; + size_t threadNum; + volatile hpatch_BOOL isDecodeFinished; + volatile hpatch_BOOL isClosing; + HCondvar finishedCond; + hpatch_dec_error_t decError; +} _lzma2mt_TDecompress; + +static void* __lzma2mt_dec_Alloc(ISzAllocPtr p, size_t size) + __dec_Alloc_fun(_lzma2mt_TDecompress,p,size) + +static SRes _lzma2mt_in_read(ISeqInStreamPtr p, void *buf, size_t *size){ + _lzma2mt_InStream* self=(_lzma2mt_InStream*)p; + if (*size==0) return SZ_OK; + if (*self->pIsClosing){ *size=0; return SZ_OK; } + { + hpatch_StreamPos_t remain=self->endPos-self->curPos; + if (*size>remain) *size=(size_t)remain; + } + if (*size==0) return SZ_OK; + if (!self->codeStream->read(self->codeStream,self->curPos, + (unsigned char*)buf,(unsigned char*)buf+*size)) + return SZ_ERROR_READ; + self->curPos+=*size; + return SZ_OK; +} + +static size_t _lzma2mt_out_write(ISeqOutStreamPtr p, const void *buf, size_t size){ + _lzma2mt_OutStream* out=(_lzma2mt_OutStream*)p; + _lzma2mt_TDecompress* self=out->dec; + const unsigned char* src=(const unsigned char*)buf; + size_t remaining=size; + while (remaining>0){ + c_locker_enter(self->ring.locker); + while ((self->ring.used==self->ring.capacity) && (!self->isClosing)){ + c_condvar_wait(self->ring.hasSpaceCond,self->ring.locker); + } + if (self->isClosing){ c_locker_leave(self->ring.locker); return 0; } + { + size_t freeSpace=self->ring.capacity-self->ring.used; + size_t toCopy=(remainingring.capacity-self->ring.w; + if (toCopy>atEnd) toCopy=atEnd; + memcpy(self->ring.buf+self->ring.w,src,toCopy); + self->ring.w=(self->ring.w+toCopy)%self->ring.capacity; + self->ring.used+=toCopy; + src+=toCopy; + remaining-=toCopy; + } + c_condvar_signal(self->ring.hasDataCond); + c_locker_leave(self->ring.locker); + } + return size; +} + +static void _lzma2mt_decode_thread(int threadIndex, void* workData){ + _lzma2mt_TDecompress* self=(_lzma2mt_TDecompress*)workData; + UInt64 inProcessed=0; + int isMT=0; + const UInt64* outSizePtr=(self->dataSize>0)?&self->dataSize:0; + SRes res=Lzma2DecMt_Decode(self->decMtHandle,self->propByte,&self->mtProps, + &self->outStream.vt,outSizePtr,1, + &self->inStream.vt,&inProcessed,&isMT,0); + if ((res!=SZ_OK) && (!self->isClosing)) + self->decError=hpatch_dec_error; + c_locker_enter(self->ring.locker); + self->isDecodeFinished=hpatch_TRUE; + c_condvar_broadcast(self->ring.hasDataCond); + c_condvar_broadcast(self->ring.hasSpaceCond); + c_condvar_signal(self->finishedCond); + c_locker_leave(self->ring.locker); +} + +static hpatch_BOOL _lzma2mt_is_can_open(const char* compressType){ + return (0==strcmp(compressType,"lzma2")); +} + +static hpatch_decompressHandle _lzma2mt_open(hpatch_TDecompress* decompressPlugin, + hpatch_StreamPos_t dataSize, + const hpatch_TStreamInput* codeStream, + hpatch_StreamPos_t code_begin, + hpatch_StreamPos_t code_end){ + _lzma2mt_TDecompress* self=0; + unsigned char propsSize=0; + if (code_end-code_begin<1) _dec_openErr_rt(); + if (!codeStream->read(codeStream,code_begin,&propsSize,&propsSize+1)) return 0; + ++code_begin; + + self=(_lzma2mt_TDecompress*)_dec_malloc(sizeof(_lzma2mt_TDecompress)); + if (!self) _dec_memErr_rt(); + memset(self,0,sizeof(_lzma2mt_TDecompress)); + + self->memAllocBase.Alloc=__lzma2mt_dec_Alloc; + *((void**)&self->memAllocBase.Free)=(void*)__dec_free; + + self->ring.buf=(unsigned char*)_dec_malloc(kLzma2mtRingBufSize); + if (!self->ring.buf){ free(self); _dec_memErr_rt(); } + self->ring.capacity=kLzma2mtRingBufSize; + self->ring.r=self->ring.w=self->ring.used=0; + self->ring.locker=c_locker_new(); + self->ring.hasSpaceCond=c_condvar_new(); + self->ring.hasDataCond=c_condvar_new(); + if ((!self->ring.locker) || (!self->ring.hasSpaceCond) || (!self->ring.hasDataCond)) + goto _lzma2mt_open_err; + + self->inStream.vt.Read=_lzma2mt_in_read; + self->inStream.codeStream=codeStream; + self->inStream.curPos=code_begin; + self->inStream.endPos=code_end; + self->inStream.pIsClosing=&self->isClosing; + + self->outStream.vt.Write=_lzma2mt_out_write; + self->outStream.dec=self; + + self->decMtHandle=Lzma2DecMt_Create(&self->memAllocBase,&self->memAllocBase); + if (!self->decMtHandle) goto _lzma2mt_open_err; + + self->propByte=propsSize; + self->dataSize=dataSize; + self->threadNum=decompressPlugin->dec_threadNum; + if (self->threadNum<1) self->threadNum=1; + + Lzma2DecMtProps_Init(&self->mtProps); + self->mtProps.numThreads=(unsigned)self->threadNum; + self->isDecodeFinished=hpatch_FALSE; + self->isClosing=hpatch_FALSE; + self->finishedCond=c_condvar_new(); + if (!self->finishedCond) goto _lzma2mt_open_err; + + if (!c_thread_parallel(1,_lzma2mt_decode_thread,self,0,0)){ + c_condvar_delete(self->finishedCond); self->finishedCond=0; + goto _lzma2mt_open_err; + } + return self; + +_lzma2mt_open_err: + if (self->ring.locker) c_locker_delete(self->ring.locker); + if (self->ring.hasSpaceCond) c_condvar_delete(self->ring.hasSpaceCond); + if (self->ring.hasDataCond) c_condvar_delete(self->ring.hasDataCond); + if (self->ring.buf) free(self->ring.buf); + if (self->decMtHandle) Lzma2DecMt_Destroy(self->decMtHandle); + if (self->finishedCond) c_condvar_delete(self->finishedCond); + free(self); + _dec_openErr_rt(); + return 0; +} + +static hpatch_BOOL _lzma2mt_close(hpatch_TDecompress* decompressPlugin, + hpatch_decompressHandle decompressHandle){ + _lzma2mt_TDecompress* self=(_lzma2mt_TDecompress*)decompressHandle; + if (!self) return hpatch_TRUE; + self->isClosing=hpatch_TRUE; + *self->inStream.pIsClosing=hpatch_TRUE; + // wake up any blocked threads + c_locker_enter(self->ring.locker); + c_condvar_broadcast(self->ring.hasSpaceCond); + c_condvar_broadcast(self->ring.hasDataCond); + c_locker_leave(self->ring.locker); + // wait for decode thread to finish + if (!self->isDecodeFinished){ + c_locker_enter(self->ring.locker); + while (!self->isDecodeFinished) + c_condvar_wait(self->finishedCond,self->ring.locker); + c_locker_leave(self->ring.locker); + } + if (self->decMtHandle) + Lzma2DecMt_Destroy(self->decMtHandle); + if (self->ring.locker) c_locker_delete(self->ring.locker); + if (self->ring.hasSpaceCond) c_condvar_delete(self->ring.hasSpaceCond); + if (self->ring.hasDataCond) c_condvar_delete(self->ring.hasDataCond); + if (self->ring.buf) free(self->ring.buf); + if (self->finishedCond) c_condvar_delete(self->finishedCond); + _dec_onDecErr_up(); + free(self); + return hpatch_TRUE; +} + +static hpatch_BOOL _lzma2mt_decompress_part(hpatch_decompressHandle decompressHandle, + unsigned char* out_part_data,unsigned char* out_part_data_end){ + _lzma2mt_TDecompress* self=(_lzma2mt_TDecompress*)decompressHandle; + unsigned char* out_cur=out_part_data; + while (out_curring.locker); + while (self->ring.used==0 && !self->isDecodeFinished){ + c_condvar_wait(self->ring.hasDataCond,self->ring.locker); + } + if (self->ring.used>0){ + size_t need=(size_t)(out_part_data_end-out_cur); + size_t toCopy=needring.used?need:self->ring.used; + size_t atEnd=self->ring.capacity-self->ring.r; + if (toCopy>atEnd) toCopy=atEnd; + memcpy(out_cur,self->ring.buf+self->ring.r,toCopy); + self->ring.r=(self->ring.r+toCopy)%self->ring.capacity; + self->ring.used-=toCopy; + out_cur+=toCopy; + c_condvar_signal(self->ring.hasSpaceCond); + c_locker_leave(self->ring.locker); + }else{ + c_locker_leave(self->ring.locker); + _dec_onDecErr_rt(); + } + } + return hpatch_TRUE; +} + +static hpatch_TDecompress lzma2mtDecompressPlugin={_lzma2mt_is_can_open,_lzma2mt_open, + _lzma2mt_close,_lzma2mt_decompress_part}; +#endif//_CompressPlugin_lzma2mt + + #ifdef _CompressPlugin_7zXZ #if (_IsNeedIncludeDefaultCompressHead) # include "Xz.h" // "lzma/C/Xz.h" https://github.com/sisong/lzma diff --git a/dirDiffPatch/dir_patch/dir_patch.c b/dirDiffPatch/dir_patch/dir_patch.c index 5638a11b..475598b5 100644 --- a/dirDiffPatch/dir_patch/dir_patch.c +++ b/dirDiffPatch/dir_patch/dir_patch.c @@ -602,8 +602,11 @@ hpatch_BOOL TDirPatcher_patch(TDirPatcher* self,const hpatch_TStreamOutput* out_ patchCacheSize_min+=(size_t)self->dirDiffInfo.sdiffInfo.stepMemSize; #endif #if (_IS_NEED_WINDOW_DIFF) - if (self->dirDiffInfo.isWindowDiff) - patchCacheSize_min+=(size_t)(self->dirDiffInfo.winDiffInfo.maxWindowOldSize+self->dirDiffInfo.winDiffInfo.maxStepMemSize); + if (self->dirDiffInfo.isWindowDiff){ + hpatch_StreamPos_t wsSize=patchCacheSize_min+self->dirDiffInfo.winDiffInfo.maxWindowOldSize+self->dirDiffInfo.winDiffInfo.maxStepMemSize; + check(wsSize==(hpatch_StreamPos_t)(size_t)wsSize); + patchCacheSize_min=(size_t)wsSize; + } #endif check(patchCacheSize_min<=(size_t)(temp_cache_end-temp_cache)); if (self->_checksumSet.isCheck_oldRefData){ diff --git a/file_for_patch.c b/file_for_patch.c index 4786d0bc..a9501567 100644 --- a/file_for_patch.c +++ b/file_for_patch.c @@ -125,12 +125,18 @@ int hpatch_printPath_utf8(const char* pathTxt_utf8){ #endif } +#if (_IS_USED_WIN32_UTF8_WAPI) +int _hpatch_printStdErrPath_wstr(const wchar_t* pathTxt_wchar){ + return LOG_ERR("%ls",pathTxt_wchar); +} +#endif + int hpatch_printStdErrPath_utf8(const char* pathTxt_utf8){ #if (_IS_USED_WIN32_UTF8_WAPI) wchar_t pathTxt_w[hpatch_kPathMaxSize]; int wsize=_utf8FileName_to_w(pathTxt_utf8,pathTxt_w,hpatch_kPathMaxSize); if (wsize>0) - return LOG_ERR("%ls",pathTxt_w); + return _hpatch_printStdErrPath_wstr(pathTxt_w); else //view unknow return LOG_ERR("%s",pathTxt_utf8); #else @@ -518,6 +524,17 @@ hpatch_FileHandle __import_fileOpen(const char* fileName_utf8,_FileModeType mode hpatch_force_inline static hpatch_FileHandle _import_fileOpen(const char* fileName_utf8,_FileModeType mode){ hpatch_FileHandle result=__import_fileOpen(fileName_utf8,mode); + if (result==0){ + #if (_IS_USED_WIN32_UTF8_WAPI) + LOG_ERR("fopen fail, errno:%d, openmode:\"",errno); + _hpatch_printStdErrPath_wstr(mode); + LOG_ERR("\", filename:\""); + hpatch_printStdErrPath_utf8(fileName_utf8); + LOG_ERR("\"\n"); + #else + LOG_ERR("fopen fail, errno:%d, openmode:\"%s\", filename:\"%s\"\n",errno,mode,fileName_utf8); + #endif + } //used default vbuf? if (result) setvbuf(result,0,_IONBF,0); return result; } diff --git a/hdiffz.cpp b/hdiffz.cpp index e3696238..97f8f316 100644 --- a/hdiffz.cpp +++ b/hdiffz.cpp @@ -296,8 +296,10 @@ static void printUsage(){ # endif #endif #ifdef _CompressPlugin_lzma2 - " -c-lzma2[-{0..9}[-dictSize]] DEFAULT level 7\n" - " dictSize can like 4096 or 4k or 4m or 128m etc..., DEFAULT 8m\n" + " -c-lzma2[-{0..9}[-dictSize[-blockSize]]] DEFAULT level 7\n" + " dictSize can like 4096 or 4k or 4m or 128m etc..., DEFAULT 8m;\n" + " 1m<=blockSize<=256m, DEFAULT 0 (AUTO) ...;\n" + " recommend blockSize==dictSize when need decompress by multi-thread;\n" # if (_IS_USED_MULTITHREAD) " support run by multi-thread parallel, fast!\n" # endif @@ -673,7 +675,8 @@ static hpatch_BOOL _getOptChecksum(hpatch_TChecksum** out_checksumPlugin, static bool _tryGetCompressSet(const char** isMatchedType,const char* ptype,const char* ptypeEnd, const char* cmpType,const char* cmpType2=0, size_t* compressLevel=0,size_t levelMin=0,size_t levelMax=0,size_t levelDefault=0, - size_t* dictSize=0,size_t dictSizeMin=0,size_t dictSizeMax=0,size_t dictSizeDefault=0){ + size_t* dictSize=0,size_t dictSizeMin=0,size_t dictSizeMax=0,size_t dictSizeDefault=0, + size_t* blockSize=0){ assert (0==(*isMatchedType)); const size_t ctypeLen=strlen(cmpType); const size_t ctype2Len=(cmpType2!=0)?strlen(cmpType2):0; @@ -695,9 +698,18 @@ static bool _tryGetCompressSet(const char** isMatchedType,const char* ptype,cons if (!kmg_to_size(pdictSize,pdictSizeEnd-pdictSize,dictSize)) return false; //error if (*dictSizedictSizeMax) *dictSize=dictSizeMax; + if (blockSize && (pdictSizeEnd[0]=='-')){ + const char* pblockSize=pdictSizeEnd+1; + const char* pblockSizeEnd=findUntilEnd(pblockSize,'-'); + if (pblockSizeEnd[0]!='\0') return false; + if (!kmg_to_size(pblockSize,pblockSizeEnd-pblockSize,blockSize)) return false; + }else if (pdictSizeEnd[0]!='\0'){ + return false; + } }else{ if (plevelEnd[0]!='\0') return false; //error if (dictSize) *dictSize=(dictSizeDefault1 then open multi-thread Parallel mode;\n" " now support window diffData(created by hdiffz -WD) and single compressed diffData(created by hdiffz -SD);\n" " can set 1..5, DEFAULT -p-1!\n" + " -p-dec-decThreadNumber\n" + " if decThreadNumber>1 then open multi-thread decompress mode, DEFAULT -p-dec-1!\n" + " now only support lzma2 with blockSize; NOTE: need more memory!\n" #endif #if ((_IS_NEED_DIR_DIFF_PATCH)||(_IS_NEED_VCDIFF)||(_IS_NEED_WINDOW_DIFF)) " -C-checksumSets\n" @@ -354,12 +357,12 @@ typedef enum THPatchResult { int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFileName, hpatch_BOOL isLoadOldAll,size_t patchCacheSize,hpatch_StreamPos_t diffDataOffset, - hpatch_StreamPos_t diffDataSize,TPatchChecksumSet* checksumSet,size_t threadNum); + hpatch_StreamPos_t diffDataSize,TPatchChecksumSet* checksumSet,size_t threadNum,size_t dec_threadNum); #if (_IS_NEED_DIR_DIFF_PATCH) int hpatch_dir(const char* oldPath,const char* diffFileName,const char* outNewPath, hpatch_BOOL isLoadOldAll,size_t patchCacheSize,size_t kMaxOpenFileNumber, TPatchChecksumSet* checksumSet,IHPatchDirListener* hlistener, - hpatch_StreamPos_t diffDataOffset,hpatch_StreamPos_t diffDataSize,size_t threadNum); + hpatch_StreamPos_t diffDataOffset,hpatch_StreamPos_t diffDataSize,size_t threadNum,size_t dec_threadNum); #endif #if (_IS_NEED_SFX) int createSfx(const char* selfExecuteFileName,const char* diffFileName,const char* out_sfxFileName); @@ -439,7 +442,6 @@ static hpatch_BOOL _toChecksumSet(const char* psets,TPatchChecksumSet* checksumS #define _THREAD_NUMBER_NULL _kNULL_SIZE #define _THREAD_NUMBER_DEFAULT 1 #define _THREAD_NUMBER_MAX 5 - #if (_IS_NEED_CMDLINE) #define _isSwapToPatchTag(tag) (0==strcmp("--patch",tag)) @@ -465,6 +467,7 @@ int hpatch_cmd_line(int argc, const char * argv[]){ hpatch_BOOL isOldPathInputEmpty=_kNULL_VALUE; hpatch_BOOL isRunSFX=_kNULL_VALUE; size_t threadNum=_THREAD_NUMBER_NULL; + size_t dec_threadNum=_THREAD_NUMBER_NULL; #if (_IS_NEED_SFX) const char* out_SFX=0; const char* selfExecuteFile=0; @@ -520,9 +523,15 @@ int hpatch_cmd_line(int argc, const char * argv[]){ } break; #if (_IS_USED_MULTITHREAD) case 'p':{ - const char* pnum=op+3; - _options_check((threadNum==_THREAD_NUMBER_NULL)&&(op[2]=='-'),"-p-?"); - _options_check(a_to_size(pnum,strlen(pnum),&threadNum),"-p-?"); + if ((op[2]=='-')&&(strncmp(op+3,"dec-",4)==0)){ + const char* pnum=op+3+4; + _options_check(dec_threadNum==_THREAD_NUMBER_NULL,"-p-dec-?"); + _options_check(a_to_size(pnum,strlen(pnum),&dec_threadNum),"-p-dec-?"); + }else{ + const char* pnum=op+3; + _options_check((threadNum==_THREAD_NUMBER_NULL)&&(op[2]=='-'),"-p-?"); + _options_check(a_to_size(pnum,strlen(pnum),&threadNum),"-p-?"); + } } break; #endif #if (_IS_NEED_SFX) @@ -604,6 +613,10 @@ int hpatch_cmd_line(int argc, const char * argv[]){ if (threadNum>1) _parallel_import_c_on_error=_on_mt_error; #endif + if (dec_threadNum==_THREAD_NUMBER_NULL) + dec_threadNum=_THREAD_NUMBER_DEFAULT; + if (dec_threadNum>_THREAD_NUMBER_MAX) + dec_threadNum=_THREAD_NUMBER_MAX; #if (_IS_NEED_SFX) if ((argc<=1)&&(!isRunSFX)){ hpatch_StreamPos_t _diffDataOffset=0; @@ -736,12 +749,12 @@ int hpatch_cmd_line(int argc, const char * argv[]){ #if (_IS_NEED_DIR_DIFF_PATCH) if (dirDiffInfo.isDirDiff){ return hpatch_dir(oldPath,diffFileName,outNewPath,isLoadOldAll,patchCacheSize,kMaxOpenFileNumber, - &checksumSet,&defaultPatchDirlistener,diffDataOffset,diffDataSize,threadNum); + &checksumSet,&defaultPatchDirlistener,diffDataOffset,diffDataSize,threadNum,dec_threadNum); }else #endif { return hpatch(oldPath,diffFileName,outNewPath,isLoadOldAll,patchCacheSize, - diffDataOffset,diffDataSize,&checksumSet,threadNum); + diffDataOffset,diffDataSize,&checksumSet,threadNum,dec_threadNum); } }else #if (_IS_NEED_DIR_DIFF_PATCH) @@ -765,12 +778,12 @@ int hpatch_cmd_line(int argc, const char * argv[]){ if (dirDiffInfo.isDirDiff){ result=hpatch_dir(oldPath,diffFileName,newTempName,isLoadOldAll,patchCacheSize, kMaxOpenFileNumber,&checksumSet,&defaultPatchDirlistener, - diffDataOffset,diffDataSize,threadNum); + diffDataOffset,diffDataSize,threadNum,dec_threadNum); }else #endif { result=hpatch(oldPath,diffFileName,newTempName,isLoadOldAll,patchCacheSize, - diffDataOffset,diffDataSize,&checksumSet,threadNum); + diffDataOffset,diffDataSize,&checksumSet,threadNum,dec_threadNum); } if (result==HPATCH_SUCCESS){ _return_check(hpatch_removeFile(oldPath), @@ -797,7 +810,7 @@ int hpatch_cmd_line(int argc, const char * argv[]){ HPATCH_TEMPPATH_ERROR,"getTempPathName(outNewPath)"); printf("NOTE: all in outNewPath temp directory will be move to oldDirectory after patch!\n"); result=hpatch_dir(oldPath,diffFileName,newTempDir,isLoadOldAll,patchCacheSize,kMaxOpenFileNumber, - &checksumSet,&tempDirPatchListener,diffDataOffset,diffDataSize,threadNum); + &checksumSet,&tempDirPatchListener,diffDataOffset,diffDataSize,threadNum,dec_threadNum); if (result==HPATCH_SUCCESS){ printf("all in outNewPath temp directory moved to oldDirectory!\n"); }else if(!hpatch_isPathNotExist(newTempDir)){ @@ -841,7 +854,7 @@ int hpatch_cmd_line(int argc, const char * argv[]){ #define _try_rt_dec(dec) { if (dec.is_can_open(compressType)) return &dec; } -static const hpatch_TDecompress* __find_decompressPlugin(const char* compressType){ +static const hpatch_TDecompress* __find_decompressPlugin(const char* compressType,size_t dec_threadNum){ #if ((defined(_CompressPlugin_ldef))&&_IS_NEED_decompressor_ldef_replace_zlib) _try_rt_dec(ldefDecompressPlugin); #else @@ -855,6 +868,9 @@ static const hpatch_TDecompress* __find_decompressPlugin(const char* compressTyp #ifdef _CompressPlugin_lzma _try_rt_dec(lzmaDecompressPlugin); #endif +#ifdef _CompressPlugin_lzma2mt + if (dec_threadNum>1) _try_rt_dec(lzma2mtDecompressPlugin); +#endif #ifdef _CompressPlugin_lzma2 _try_rt_dec(lzma2DecompressPlugin); #endif @@ -908,16 +924,17 @@ static hpatch_BOOL getVcDiffDecompressPlugin(hpatch_TDecompress* out_decompressP #endif static hpatch_BOOL getDecompressPlugin(const hpatch_compressedDiffInfo* diffInfo, - hpatch_TDecompress* out_decompressPlugin){ + hpatch_TDecompress* out_decompressPlugin,size_t dec_threadNum){ const hpatch_TDecompress* decompressPlugin=0; memset(out_decompressPlugin,0,sizeof(*out_decompressPlugin)); if (diffInfo->compressedCount>0){ - decompressPlugin=__find_decompressPlugin(diffInfo->compressType); + decompressPlugin=__find_decompressPlugin(diffInfo->compressType,dec_threadNum); if ((0==decompressPlugin)||(decompressPlugin->open==0)) return hpatch_FALSE; //error } if (decompressPlugin){ *out_decompressPlugin=*decompressPlugin; out_decompressPlugin->decError=hpatch_dec_ok; + out_decompressPlugin->dec_threadNum=dec_threadNum; } return hpatch_TRUE; } @@ -1004,7 +1021,7 @@ static hpatch_BOOL _win_onDiffInfo(struct winpatch_listener_t* listener,const hp hpatch_BOOL* isChecksumNew,hpatch_BOOL* isChecksumOld,hpatch_BOOL* isChecksumDiff, unsigned char** out_temp_cache,unsigned char** out_temp_cacheEnd){ _WinPatchListener_t* ctx=(_WinPatchListener_t*)listener->import; - *out_decompressPlugin=ctx->decompressPlugin; //or __find_decompressPlugin() + *out_decompressPlugin=ctx->decompressPlugin; //or getDecompressPlugin() {//checksum *out_checksumPlugin=0; *isChecksumNew=hpatch_FALSE; @@ -1065,7 +1082,7 @@ typedef struct _THDiffInfos{ #define _kUnavailableSize hpatch_kNullStreamPos -static int _getHDiffInfos(_THDiffInfos* out_diffInfos,const hpatch_TFileStreamInput* diffData){ +static int _getHDiffInfos(_THDiffInfos* out_diffInfos,const hpatch_TFileStreamInput* diffData,size_t dec_threadNum){ int result=HPATCH_SUCCESS; int _isInClear=hpatch_FALSE; hpatch_TDecompress* decompressPlugin=&out_diffInfos->_decompressPlugin; @@ -1115,7 +1132,7 @@ static int _getHDiffInfos(_THDiffInfos* out_diffInfos,const hpatch_TFileStreamIn check(hpatch_FALSE,HPATCH_HDIFFINFO_ERROR,"is hdiff file? get diffInfo"); } if (decompressPlugin->open==0){ - if (getDecompressPlugin(diffInfo,decompressPlugin)){ + if (getDecompressPlugin(diffInfo,decompressPlugin,dec_threadNum)){ }else{ LOG_ERR("can not decompress \"%s\" data ERROR!\n",out_diffInfos->diffInfo.compressType); check_on_error(HPATCH_COMPRESSTYPE_ERROR); @@ -1253,7 +1270,7 @@ static int _printFileInfos(const char* fileName,const char* fileTag){ #endif if (!isDirDiff){ _THDiffInfos diffInfos={0}; - result=_getHDiffInfos(&diffInfos,&diffData); + result=_getHDiffInfos(&diffInfos,&diffData,1); if ((result!=HPATCH_SUCCESS)&&(result!=HPATCH_COMPRESSTYPE_ERROR)) check_on_error(result); _printHDiffInfos(&diffInfos,isDirDiff); @@ -1356,7 +1373,7 @@ static TByte* allocPatchMemCache(hpatch_StreamPos_t kMinCacheSize,hpatch_StreamP int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFileName, hpatch_BOOL isLoadOldAll,size_t patchCacheSize,hpatch_StreamPos_t diffDataOffset, - hpatch_StreamPos_t diffDataSize,TPatchChecksumSet* checksumSet,size_t threadNum){ + hpatch_StreamPos_t diffDataSize,TPatchChecksumSet* checksumSet,size_t threadNum,size_t dec_threadNum){ int result=HPATCH_SUCCESS; int _isInClear=hpatch_FALSE; double time0=clock_s(); @@ -1403,7 +1420,7 @@ int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFi printf(" diffDataSize: %" PRIu64 "\n",diffData.base.streamSize); {//info - int ret=_getHDiffInfos(&diffInfos,&diffData); + int ret=_getHDiffInfos(&diffInfos,&diffData,dec_threadNum); #if (_IS_NEED_PRINT_LOG) if ((ret!=HPATCH_SUCCESS)&&(ret!=HPATCH_COMPRESSTYPE_ERROR)) check_on_error(ret); @@ -1540,7 +1557,7 @@ int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFi int hpatch_dir(const char* oldPath,const char* diffFileName,const char* outNewPath, hpatch_BOOL isLoadOldAll,size_t patchCacheSize,size_t kMaxOpenFileNumber, TPatchChecksumSet* checksumSet,IHPatchDirListener* hlistener, - hpatch_StreamPos_t diffDataOffset,hpatch_StreamPos_t diffDataSize,size_t threadNum){ + hpatch_StreamPos_t diffDataOffset,hpatch_StreamPos_t diffDataSize,size_t threadNum,size_t dec_threadNum){ int result=HPATCH_SUCCESS; int _isInClear=hpatch_FALSE; double time0=clock_s(); @@ -1609,7 +1626,7 @@ int hpatch_dir(const char* oldPath,const char* diffFileName,const char* outNewPa hpatch_compressedDiffInfo hdiffInfo; hdiffInfo=dirDiffInfo->hdiffInfo; hdiffInfo.compressedCount+=(dirDiffInfo->dirDataIsCompressed)?1:0; - if(!getDecompressPlugin(&hdiffInfo,decompressPlugin)){ + if(!getDecompressPlugin(&hdiffInfo,decompressPlugin,dec_threadNum)){ LOG_ERR("can not decompress \"%s\" data ERROR!\n",hdiffInfo.compressType); check_on_error(HPATCH_COMPRESSTYPE_ERROR); } diff --git a/libHDiffPatch/HDiff/diff.cpp b/libHDiffPatch/HDiff/diff.cpp index 462786f4..c405f883 100644 --- a/libHDiffPatch/HDiff/diff.cpp +++ b/libHDiffPatch/HDiff/diff.cpp @@ -72,22 +72,21 @@ static const int kMinMatchScore = 2; //min match benefit threshold for cover sea static const hpatch_uint64_t kDefaultLimitCoverLen=((hpatch_uint64_t)1<<30); //<=2GB-1 namespace hdiff_private{ - void loadOldAndNewStream(TAutoMem& out_mem,const hpatch_TStreamInput* oldStream,hpatch_StreamPos_t oldPos,size_t old_size, - const hpatch_TStreamInput* newStream,hpatch_StreamPos_t newPos,size_t new_size){ - if (old_size>0) assert(oldPos+old_size<=oldStream->streamSize); - assert(newPos+new_size<=newStream->streamSize); - if (sizeof(size_t)sizeof(size_t)) + check((new_size==_new_size)&&(old_size==_old_size)); + check(old_size<=(size_t)((~(size_t)0)-new_size)); out_mem.realloc(old_size+new_size); - if (old_size) check(oldStream->read(oldStream,oldPos,out_mem.data(),out_mem.data()+old_size)); - check(newStream->read(newStream,newPos,out_mem.data()+old_size,out_mem.data()+old_size+new_size)); + if (old_size) check(oldStream->read(oldStream,old_off,out_mem.data(),out_mem.data()+old_size)); + check(newStream->read(newStream,new_off,out_mem.data()+old_size,out_mem.data()+old_size+new_size)); } void loadOldAndNewStream(TAutoMem& out_mem,const hpatch_TStreamInput* oldStream,const hpatch_TStreamInput* newStream){ _out_diff_info(" load all datas into memory from old & new ...\n"); - if (oldStream) check((size_t)oldStream->streamSize==(size_t)oldStream->streamSize); - check((size_t)newStream->streamSize==(size_t)newStream->streamSize); - loadOldAndNewStream(out_mem, oldStream,0,oldStream?(size_t)oldStream->streamSize:0, - newStream,0,(size_t)newStream->streamSize); + loadOldAndNewStream(out_mem, oldStream,0,oldStream?oldStream->streamSize:0, + newStream,0,newStream->streamSize); } } @@ -1362,7 +1361,8 @@ void get_match_covers_by_stream_and_sstring(const hpatch_TStreamInput* newData,c int kMinSingleMatchScore,bool isUseBigCacheMatch,const hdiff_TMTSets_s* mtsets, bool isExtendCover,TCachedNewOldStreams* out_cachedStreams){ mtsets=mtsets?mtsets:&hdiff_TMTSets_s_kEmpty; - if (fastMatchBlockSize==0){ + bool isRemoveOldInvalid=(fastMatchBlockSize>0)||isUseBigCacheMatch; + if ((fastMatchBlockSize==0)&&(!isRemoveOldInvalid)){ TCachedNewOldStreams cachedStreams; cachedStreams.freeCached=_free_TCachedMemStreams; cachedStreams.import=new _TCachedMemStreams(); @@ -1386,7 +1386,7 @@ void get_match_covers_by_stream_and_sstring(const hpatch_TStreamInput* newData,c TCachedNewOldStreams cachedStreams; cachedStreams.freeCached=_free_TCoversOptimStream; - cachedStreams.import=new TCoversOptimStream(newData,oldData,fastMatchBlockSize,mtsets->threadNum,mtsets->threadNumForSearch); + cachedStreams.import=new TCoversOptimStream(newData,oldData,fastMatchBlockSize,mtsets,isRemoveOldInvalid); TCoversOptimStream& coversOp=*(TCoversOptimStream*)cachedStreams.import; get_match_covers_by_sstring(coversOp.matchBlock->newData,coversOp.matchBlock->newData_end_cur, coversOp.matchBlock->oldData,coversOp.matchBlock->oldData_end_cur, @@ -1403,12 +1403,14 @@ void get_match_covers_by_stream_and_sstring(unsigned char* newData,unsigned char std::vector& out_covers,size_t fastMatchBlockSize, int kMinSingleMatchScore,bool isUseBigCacheMatch, size_t threadNum,bool isExtendCover){ - if (fastMatchBlockSize==0){ + bool isRemoveOldInvalid=(fastMatchBlockSize>0)||isUseBigCacheMatch; + if ((fastMatchBlockSize==0)&&(!isRemoveOldInvalid)){ get_match_covers_by_sstring(newData,newData_end,oldData,oldData_end,out_covers, kMinSingleMatchScore,isUseBigCacheMatch,threadNum,isExtendCover); return; } - TCoversOptimMem coversOp(newData,newData_end,oldData,oldData_end,fastMatchBlockSize,threadNum); + TCoversOptimMem coversOp(newData,newData_end,oldData,oldData_end,fastMatchBlockSize, + threadNum,isRemoveOldInvalid); get_match_covers_by_sstring(coversOp.matchBlock->newData,coversOp.matchBlock->newData_end_cur, coversOp.matchBlock->oldData,coversOp.matchBlock->oldData_end_cur, out_covers,kMinSingleMatchScore,isUseBigCacheMatch, @@ -2179,8 +2181,7 @@ void get_match_covers_in_a_window(const hpatch_TStreamInput* newData,const hpatc } TAutoMem mem; - loadOldAndNewStream(mem,oldData,window.oldPos,(size_t)window.oldLength, - newData,window.newPos,(size_t)window.newLength); + loadOldAndNewStream(mem,oldData,window.oldPos,window.oldLength,newData,window.newPos,window.newLength); unsigned char* pOldData=mem.data(); unsigned char* pNewData=pOldData+window.oldLength; if (!bigCovers.empty()) { diff --git a/libHDiffPatch/HDiff/private_diff/limit_mem_diff/stream_serialize.cpp b/libHDiffPatch/HDiff/private_diff/limit_mem_diff/stream_serialize.cpp index 571569dc..c7fcdacc 100644 --- a/libHDiffPatch/HDiff/private_diff/limit_mem_diff/stream_serialize.cpp +++ b/libHDiffPatch/HDiff/private_diff/limit_mem_diff/stream_serialize.cpp @@ -1096,6 +1096,7 @@ TWindowDiffStream::TWindowDiffStream(const hpatch_TStreamInput* newStream,const outMaxWindowOldLength=0; for (size_t wi=0;wi #include //std::runtime_error +#include "../../../libParallel/parallel_channel.h" //CHLocker, CAutoLocker +#if (_IS_USED_MULTITHREAD) +#include +#endif #define _check(value,info) { if (!(value)) { throw std::runtime_error(info); } } namespace hdiff_private { @@ -50,6 +54,7 @@ namespace hdiff_private { template static void _getPackedCovers(hpatch_StreamPos_t dataSize,const std::vector& blockCovers, std::vector& out_packedCovers){ + out_packedCovers.clear(); const hpatch_TCover* cover=blockCovers.data(); const hpatch_TCover* cover_end=cover+blockCovers.size(); hpatch_StreamPos_t dst=0; @@ -77,23 +82,301 @@ namespace hdiff_private { //return dst; } + + struct _range_less_by_begin_t{ + inline bool operator()(const hdiff_TRange& a, const hdiff_TRange& b) const { + return a.beginPos < b.beginPos; + } + }; + + static void _toMatchedRanges(std::vector& matchedRanges,const std::vector& blockCovers){ + matchedRanges.resize(blockCovers.size()); + for (size_t i=0;i& blockCovers, + size_t threadNum,bool oldDataIsMTSafe, + size_t _minOldInvalidSize,size_t _kBloomZoom,size_t _rollLen, + size_t _R,size_t _hitRateThreshold,size_t _cacheBlockSize) + :minOldInvalidSize(_minOldInvalidSize),kBloomZoom(_kBloomZoom),rollLen(_rollLen), + R(_R),hitRateThreshold(_hitRateThreshold),cacheBlockSize(_cacheBlockSize),oldSize(oldStream?oldStream->streamSize:0){ + if (((size_t)(newData_end-newData)>=rollLen)&&(oldSize>=rollLen)&&(minOldInvalidSize>0)){ + _out_diff_info(" build cache for search invalid oldData ranges ...\n"); + bloomFilter.buildMatchCache(newData,newData_end,threadNum,kBloomZoom,rollLen); + _toMatchedRanges(matchedRanges,blockCovers); + _out_diff_info(" search invalid oldData ranges for skip ...\n"); + _scanAndSmooth(oldStream,threadNum,oldDataIsMTSafe); + } +} + +struct _TOldInvalidScanCtx{ + const hpatch_TStreamInput* oldStream; + const TFastMatchForSString* bloomFilter; + hpatch_StreamPos_t oldSize; + size_t R,rollLen,hitRateThreshold,minOldInvalidSize,cacheBlockSize; + const std::vector* matchedRanges; +#if (_IS_USED_MULTITHREAD) + hpatch_StreamPos_t nextPos; + size_t rangeIdx; + std::vector* allInvalidRanges; + CHLocker taskLocker; + CHLocker mergeLocker; + CHLocker* readLocker; +#endif +}; + +static bool _getNextScanTask(_TOldInvalidScanCtx& ctx,hpatch_StreamPos_t& _pos, size_t& _rangeIdx, + hpatch_StreamPos_t& outBegin, hpatch_StreamPos_t& outEnd){ + const std::vector& matchedRanges = *ctx.matchedRanges; + const size_t matchedRanges_size=matchedRanges.size(); + hpatch_StreamPos_t pos = _pos; + size_t rangeIdx=_rangeIdx; + while (pos=matchedRanges[rangeIdx].beginPos)){ + pos=matchedRanges[rangeIdx].endPos; + ++rangeIdx; + } + hpatch_StreamPos_t segEnd=(rangeIdx& localRanges){ + const size_t kWin=R*2+1; + bool isInvalid = (hitCount*256 < kWin*ctx.hitRateThreshold); + if (isInvalid){ + if (curInvalidStart==hpatch_kNullStreamPos) + curInvalidStart=curOldPos; + } else { + if (curInvalidStart!=hpatch_kNullStreamPos){ + hdiff_TRange r={curInvalidStart,curOldPos}; + if (((hpatch_StreamPos_t)(r.endPos-r.beginPos)>=ctx.minOldInvalidSize)||(curInvalidStart==segOldPos)) + localRanges.push_back(r); + curInvalidStart=hpatch_kNullStreamPos; + } + } + const size_t ri =(size_t)((hpatch_StreamPos_t)(curOldPos-segOldPos+R+1)%kWin); + hitCount+= (size_t)hit - ring[ri]; + ring[ri] = hit; + } + +static void _scanSegment(_TOldInvalidScanCtx& ctx,const unsigned char* buf, size_t bufSize,unsigned char* ring, + hpatch_StreamPos_t segOldPos,std::vector& localRanges){ + hpatch_StreamPos_t curOldPos=segOldPos; + const size_t R=ctx.R, rollLen=ctx.rollLen, kWin=R*2+1; + if (bufSizeisHit(h); + cur+=rollLen; + size_t hitCount=ring[0]; + for (size_t ri=1; ri<=R; ++ri,++cur) {//front border + h=TFastMatchForSString::rollHash(h,cur,rollLen); + unsigned char hit=(unsigned char)ctx.bloomFilter->isHit(h); + ring[kWin-ri]=hit; + ring[ri]=hit; + hitCount+=hit*2; + } + + for (;curisHit(h); + _processHitAndUpdateRing(ctx,curInvalidStart,hitCount,hit,curOldPos,segOldPos,R,ring,localRanges); + } + + {//back border + const size_t ri_last=(size_t)((hpatch_StreamPos_t)((curOldPos-1)-segOldPos+R+1)%kWin)+kWin; + for (size_t ri=ri_last-1;ri>=ri_last-R;--ri,++curOldPos){ + unsigned char hit=ring[ri%kWin]; + _processHitAndUpdateRing(ctx,curInvalidStart,hitCount,hit,curOldPos,segOldPos,R,ring,localRanges); + } + } + if (curInvalidStart!=hpatch_kNullStreamPos){//last invalid range + hdiff_TRange r = {curInvalidStart,segOldPos+bufSize}; + localRanges.push_back(r); + } +} + +#if (_IS_USED_MULTITHREAD) +static void _scanWorker(_TOldInvalidScanCtx* ctx,unsigned char* buf){ + std::vector localRanges; + localRanges.reserve(ctx->cacheBlockSize/(ctx->minOldInvalidSize*2)); + while (true) { + hpatch_StreamPos_t taskBegin,taskEnd; + { + CAutoLocker _autoLocker(ctx->taskLocker); + if (!_getNextScanTask(*ctx,ctx->nextPos,ctx->rangeIdx,taskBegin,taskEnd)) + break; + } + size_t segLen=(size_t)(taskEnd-taskBegin); + assert(segLen<=ctx->cacheBlockSize); + { + CAutoLocker _autoLocker(*ctx->readLocker); + _check(ctx->oldStream->read(ctx->oldStream, taskBegin,buf,buf+segLen), + "TOldInvalidFilter::_scanWorker() oldStream read error!"); + } + localRanges.clear(); + _scanSegment(*ctx,buf,segLen,buf+segLen,taskBegin,localRanges); + if (!localRanges.empty()){ + CAutoLocker _autoLocker(ctx->mergeLocker); + ctx->allInvalidRanges->insert(ctx->allInvalidRanges->end(),localRanges.begin(),localRanges.end()); + } + } +} +#endif + +void TOldInvalidFilter::_scanAndSmooth(const hpatch_TStreamInput* oldStream,size_t threadNum,bool oldDataIsMTSafe){ + if (oldSize1) && (oldSize/2>=_cacheBlockSize)) { + while ((threadNum>2)&&(oldSize/threadNum<_cacheBlockSize)) --threadNum; + _check((_cacheBlockSize+kWin)<=(~(size_t)0)/threadNum,"cacheBlockSize or threadNum too big!"); + TAutoMem localCache((_cacheBlockSize+kWin)*threadNum); + CHLocker readLocker(!oldDataIsMTSafe); + ctx.readLocker = &readLocker; + ctx.nextPos = 0; + ctx.rangeIdx = 0; + ctx.allInvalidRanges = &invalidRanges; + + const size_t workerCount = threadNum - 1; + std::vector threads(workerCount); + for (size_t i=0; i=segLen); + _check(oldStream->read(oldStream,taskBegin,buf,buf+segLen), + "TOldInvalidFilter::_scanAndSmooth() oldStream read error!"); + _scanSegment(ctx,buf,segLen,buf+segLen,taskBegin,invalidRanges); + } + } + + { + if (invalidRanges.size()>1){ + std::sort(invalidRanges.begin(),invalidRanges.end(),_range_less_by_begin_t()); + size_t backi=0; + for (size_t i=1;i=minOldInvalidSize) + invalidRanges[insert++]=invalidRanges[i]; + } + invalidRanges.resize(insert); + } +} + void TMatchBlockMem::getBlockCovers(){ + if (matchBlockSize==0) return; get_match_covers_by_stream(newData,newData_end,oldData,oldData_end, blockCovers,matchBlockSize,threadNum); } void TMatchBlockStream::getBlockCovers(){ - const hdiff_TMTSets_s mtsets={threadNum,threadNumForStream,false,false}; + if (matchBlockSize==0) return; get_match_covers_by_stream(newStream,oldStream,blockCovers,matchBlockSize,&mtsets); } -void TMatchBlockBase::_getPackedCover(hpatch_StreamPos_t newDataSize,hpatch_StreamPos_t oldDataSize){ + +void TMatchBlockBase::_getOldPackedCover(hpatch_StreamPos_t oldDataSize){ + const size_t blockCount=blockCovers.size(); + if (!invalidOldRanges.empty()){ + //append invalidOldRanges as virtual covers with invalid newPos, to exclude them from packedCoversForOld + const hpatch_StreamPos_t kInvalidMaxNewPos=~(hpatch_StreamPos_t)0; + for (size_t i=0;i()); _getPackedCovers(oldDataSize,blockCovers,packedCoversForOld); std::sort(blockCovers.begin(),blockCovers.end(),cover_cmp_by_new_t()); + blockCovers.resize(blockCount); +} +void TMatchBlockBase::_getNewPackedCover(hpatch_StreamPos_t newDataSize){ _getPackedCovers(newDataSize,blockCovers,packedCoversForNew); } + static inline hpatch_StreamPos_t _getPackedSize(const std::vector& packedCovers){ + const TPackedCover* cv=packedCovers.empty()?0:&packedCovers[packedCovers.size()-1]; + return cv?cv->newPos+cv->length:0; + } +hpatch_StreamPos_t TMatchBlockBase::_getTempOldPackedSize(hpatch_StreamPos_t oldDataSize){ + assert(invalidOldRanges.empty()); + _getOldPackedCover(oldDataSize); + hpatch_StreamPos_t result=_getPackedSize(packedCoversForOld); + packedCoversForOld.clear(); + return result; +} static unsigned char* doPackData(unsigned char* data,unsigned char* data_end, const std::vector& packedCovers){ @@ -106,16 +389,26 @@ void TMatchBlockBase::_getPackedCover(hpatch_StreamPos_t newDataSize,hpatch_Stre } return dst; } + static inline bool _isBigOldData(hpatch_StreamPos_t oldSize,hpatch_StreamPos_t newSize){ + return oldSize>std::max((hpatch_StreamPos_t)1<<20,newSize/((oldSize>=(hpatch_StreamPos_t)2<<30)?6:3)); + } void TMatchBlockMem::packData(){ - if (blockCovers.empty()) return; - oldData_end_cur=doPackData(oldData,oldData_end,packedCoversForOld); + _getNewPackedCover(newData_end-newData); newData_end_cur=doPackData(newData,newData_end,packedCoversForNew); -} - static inline hpatch_StreamPos_t _getPackedSize(const std::vector& packedCovers){ - const TPackedCover* cv=packedCovers.empty()?0:&packedCovers[packedCovers.size()-1]; - return cv?cv->newPos+cv->length:0; + invalidOldRanges.clear(); + if ((isRemoveOldInvalid && _isBigOldData(_getTempOldPackedSize(oldData_end-oldData),newData_end_cur-newData))){ + hpatch_TStreamInput oldStream; + mem_as_hStreamInput(&oldStream,oldData,oldData_end); + TOldInvalidFilter filter(newData,newData_end_cur,&oldStream,blockCovers,threadNum,true); + invalidOldRanges.swap(filter.getInvalidRanges()); + minOldInvalidSize=filter.getMinOldInvalidSize(); } + _getOldPackedCover(oldData_end-oldData); + _clearV(invalidOldRanges); + oldData_end_cur=doPackData(oldData,oldData_end,packedCoversForOld); +} + static bool loadPackData(unsigned char* dst_begin,unsigned char* dst_end, const hpatch_TStreamInput* srcStream,const std::vector& packedCovers){ unsigned char* dst=dst_begin; @@ -129,32 +422,42 @@ void TMatchBlockMem::packData(){ return true; } void TMatchBlockStream::packData(){ - //load packed new & old data - const hpatch_StreamPos_t packedOldSize=_getPackedSize(packedCoversForOld); + _getNewPackedCover(newStream->streamSize); const hpatch_StreamPos_t packedNewSize=_getPackedSize(packedCoversForNew); - _check(packedNewSize==(size_t)packedNewSize,"TMatchBlockStream::packData() packedNewSize"); - if (packedOldSize) _check(packedNewSize+packedOldSize==(size_t)(packedNewSize+packedOldSize),"TMatchBlockStream::packData() packedOldSize"); - _packedNewOldMem->realloc((size_t)(packedNewSize+packedOldSize)); - oldData=_packedNewOldMem->data(); - oldData_end_cur=oldData+packedOldSize; - newData=oldData_end_cur; + _check(packedNewSize==(size_t)packedNewSize,"TMatchBlockStream::packData() packedNewSize too big!"); + _packedNewMem.realloc((size_t)packedNewSize); + newData=_packedNewMem.data(); newData_end_cur=newData+packedNewSize; - _out_diff_info(" load datas into memory from old & new ...\n"); - _check(loadPackData(oldData,oldData_end_cur,oldStream,packedCoversForOld),"loadPackData(oldStream)"); - _check(loadPackData(newData,newData_end_cur,newStream,packedCoversForNew),"loadPackData(newStream)"); + _out_diff_info(" load new data into memory from new stream ...\n"); + _check(loadPackData(newData,newData_end_cur,newStream,packedCoversForNew),"loadPackData(newStream) newStream read error!"); + + invalidOldRanges.clear(); + if (isRemoveOldInvalid&&_isBigOldData(_getTempOldPackedSize(oldStream->streamSize),newData_end_cur-newData)){ + TOldInvalidFilter filter(newData,newData_end_cur,oldStream,blockCovers,mtsets.threadNum,mtsets.oldDataIsMTSafe); + invalidOldRanges.swap(filter.getInvalidRanges()); + minOldInvalidSize=filter.getMinOldInvalidSize(); + } + _getOldPackedCover(oldStream->streamSize); + _clearV(invalidOldRanges); + const hpatch_StreamPos_t packedOldSize=_getPackedSize(packedCoversForOld); + _check(packedOldSize==(size_t)packedOldSize,"TMatchBlockStream::packData() packedOldSize too big!"); + _packedOldMem.realloc((size_t)packedOldSize); + oldData=_packedOldMem.data(); + oldData_end_cur=oldData+packedOldSize; + _out_diff_info(" load old data into memory from old stream ...\n"); + _check(loadPackData(oldData,oldData_end_cur,oldStream,packedCoversForOld),"loadPackData(oldStream) oldStream read error!"); } TMatchBlockStream::TMatchBlockStream(const hpatch_TStreamInput* _newStream,const hpatch_TStreamInput* _oldStream, - size_t _matchBlockSize,size_t _threadNumForMem,size_t _threadNumForStream) -:TMatchBlockBase(_matchBlockSize,_threadNumForMem), + size_t _matchBlockSize,const hdiff_TMTSets_s* _mtsets,bool _isRemoveOldInvalid) +:TMatchBlockBase(_matchBlockSize,_mtsets->threadNum), newData(0),newData_end_cur(0),oldData(0),oldData_end_cur(0),newStream(_newStream),oldStream(_oldStream), - threadNumForStream(_threadNumForStream),_newStreamMap(_newStream,packedCoversForNew), - _oldStreamMap(_oldStream,packedCoversForOld),_packedNewOldMem(0),_isUnpacked(false){ + mtsets(*_mtsets),isRemoveOldInvalid(_isRemoveOldInvalid), + _newStreamMap(_newStream,packedCoversForNew), + _oldStreamMap(_oldStream,packedCoversForOld),_isUnpacked(false){ assert(_oldStream); - _packedNewOldMem=new TAutoMem(); } TMatchBlockStream::~TMatchBlockStream(){ - if (_packedNewOldMem) delete _packedNewOldMem; } template static diff --git a/libHDiffPatch/HDiff/private_diff/match_block.h b/libHDiffPatch/HDiff/private_diff/match_block.h index f309d240..5c3a4953 100644 --- a/libHDiffPatch/HDiff/private_diff/match_block.h +++ b/libHDiffPatch/HDiff/private_diff/match_block.h @@ -28,23 +28,57 @@ #ifndef hdiff_match_block_h #define hdiff_match_block_h #include "../diff_types.h" +#include "suffix_string.h" +#include "mem_buf.h" #include namespace hdiff_private{ - struct TAutoMem; - + + //identify large invalid regions in old data before diffing: + //build a bloom filter from new data's rolling hashes, + //stream old data positions against the filter through a small ring buffer, + //applying a sliding-window hit-rate threshold to output invalid ranges. + struct TOldInvalidFilter{ + TOldInvalidFilter(const unsigned char* newData,const unsigned char* newData_end, + const hpatch_TStreamInput* oldStream,const std::vector& blockCovers, + size_t threadNum,bool oldDataIsMTSafe, + size_t minOldInvalidSize=1024,size_t kBloomZoom=6,size_t rollLen=5, + size_t R=32, size_t hitRateThreshold=64, //hitRateThreshold percent 0-256 + size_t cacheBlockSize=(1<<20)); //1MB + inline std::vector& getInvalidRanges(){ return invalidRanges; } + inline size_t getMinOldInvalidSize()const { return minOldInvalidSize; } + private: + void _scanAndSmooth(const hpatch_TStreamInput* oldStream,size_t threadNum,bool oldDataIsMTSafe); + TFastMatchForSString bloomFilter; + std::vector invalidRanges; + std::vector matchedRanges; + const size_t minOldInvalidSize; + const size_t kBloomZoom; + const size_t rollLen; + const size_t R; + const size_t hitRateThreshold; //percent 0-256 + const size_t cacheBlockSize; + hpatch_StreamPos_t oldSize; + }; + struct TMatchBlockBase{ typedef hpatch_TCover TPackedCover; TMatchBlockBase(size_t _matchBlockSize,size_t _threadNum) - :matchBlockSize(_matchBlockSize),threadNum(_threadNum){} + :matchBlockSize(_matchBlockSize),threadNum(_threadNum),minOldInvalidSize(0){} inline void swapBlockCovers(std::vector& _blockCovers){ blockCovers.swap(_blockCovers); } protected: - void _getPackedCover(hpatch_StreamPos_t newDataSize,hpatch_StreamPos_t oldDataSize); + void _getNewPackedCover(hpatch_StreamPos_t newDataSize); + void _getOldPackedCover(hpatch_StreamPos_t oldDataSize); + hpatch_StreamPos_t _getTempOldPackedSize(hpatch_StreamPos_t oldDataSize); + virtual void _on_invalidOldRanges(std::vector& curBlockCovers,size_t invalidBegin, + size_t minOldInvalidSize){ } void _unpackData(IDiffInsertCover* diffi,hpatch_TCover*& pcovers,size_t& coverCount); const size_t matchBlockSize; const size_t threadNum; + size_t minOldInvalidSize; std::vector blockCovers; std::vector packedCoversForOld; std::vector packedCoversForNew; + std::vector invalidOldRanges; }; //remove some big match block befor diff, in memory @@ -55,16 +89,17 @@ namespace hdiff_private{ unsigned char* oldData; unsigned char* oldData_end; unsigned char* oldData_end_cur; + const bool isRemoveOldInvalid; TMatchBlockMem(unsigned char* _newData,unsigned char* _newData_end, unsigned char* _oldData,unsigned char* _oldData_end, - size_t _matchBlockSize,size_t _threadNumForMem) + size_t _matchBlockSize,size_t _threadNumForMem,bool _isRemoveOldInvalid=false) :TMatchBlockBase(_matchBlockSize,_threadNumForMem), newData(_newData),newData_end(_newData_end),newData_end_cur(_newData_end), - oldData(_oldData),oldData_end(_oldData_end),oldData_end_cur(_oldData_end){ } + oldData(_oldData),oldData_end(_oldData_end),oldData_end_cur(_oldData_end), + isRemoveOldInvalid(_isRemoveOldInvalid){ } inline hpatch_StreamPos_t curNewDataSize()const{ return (size_t)(newData_end_cur-newData); } inline hpatch_StreamPos_t curOldDataSize()const{ return (size_t)(oldData_end_cur-oldData); } void getBlockCovers(); - inline void getPackedCover() { _getPackedCover(newData_end-newData,oldData_end-oldData); } void packData(); void unpackData(IDiffInsertCover* diffi,hpatch_TCover* pcovers,size_t coverCount); }; @@ -77,14 +112,14 @@ namespace hdiff_private{ unsigned char* oldData_end_cur; const hpatch_TStreamInput* newStream; const hpatch_TStreamInput* oldStream; - const size_t threadNumForStream; + const hdiff_TMTSets_s mtsets; + const bool isRemoveOldInvalid; TMatchBlockStream(const hpatch_TStreamInput* _newStream,const hpatch_TStreamInput* _oldStream, - size_t _matchBlockSize,size_t _threadNumForMem,size_t _threadNumForStream); + size_t _matchBlockSize,const hdiff_TMTSets_s* mtsets,bool _isRemoveOldInvalid=false); ~TMatchBlockStream(); inline hpatch_StreamPos_t curNewDataSize()const{ return _isUnpacked?newStream->streamSize:(size_t)(newData_end_cur-newData); } inline hpatch_StreamPos_t curOldDataSize()const{ return _isUnpacked?oldStream->streamSize:(size_t)(oldData_end_cur-oldData); } void getBlockCovers(); - inline void getPackedCover() { _getPackedCover(newStream->streamSize,oldStream->streamSize); } void packData(); void unpackData(IDiffInsertCover* diffi,hpatch_TCover* pcovers,size_t coverCount); void cachedStreams(const hpatch_TStreamInput** pnewData,const hpatch_TStreamInput** poldData); @@ -101,7 +136,8 @@ namespace hdiff_private{ protected: TStreamInputMap _newStreamMap; TStreamInputMap _oldStreamMap; - TAutoMem* _packedNewOldMem; + TAutoMem _packedNewMem; + TAutoMem _packedOldMem; bool _isUnpacked; }; @@ -130,11 +166,10 @@ namespace hdiff_private{ struct TCoversOptimMem:public TCoversOptim{ TCoversOptimMem(unsigned char* newData,unsigned char* newData_end, unsigned char* oldData,unsigned char* oldData_end, - size_t matchBlockSize,size_t threadNum) + size_t matchBlockSize,size_t threadNum,bool isRemoveOldInvalid) :TCoversOptim(&_matchBlock), - _matchBlock(newData,newData_end,oldData,oldData_end,matchBlockSize,threadNum){ + _matchBlock(newData,newData_end,oldData,oldData_end,matchBlockSize,threadNum,isRemoveOldInvalid){ matchBlock->getBlockCovers(); - matchBlock->getPackedCover(); matchBlock->packData(); } protected: @@ -143,11 +178,10 @@ namespace hdiff_private{ struct TCoversOptimStream:public TCoversOptim{ TCoversOptimStream(const hpatch_TStreamInput* newStream,const hpatch_TStreamInput* oldStream, - size_t matchBlockSize,size_t threadNumForMem,size_t threadNumForStream) + size_t matchBlockSize,const hdiff_TMTSets_s* mtsets,bool isRemoveOldInvalid) :TCoversOptim(&_matchBlock), - _matchBlock(newStream,oldStream,matchBlockSize,threadNumForMem,threadNumForStream){ + _matchBlock(newStream,oldStream,matchBlockSize,mtsets,isRemoveOldInvalid){ matchBlock->getBlockCovers(); - matchBlock->getPackedCover(); matchBlock->packData(); } inline void cachedStreams(const hpatch_TStreamInput** pnewData,const hpatch_TStreamInput** poldData){ @@ -164,7 +198,6 @@ namespace hdiff_private{ :TCoversOptim(&_matchBlock), _matchBlock(newData,newData_end,oldData,oldData_end,0,threadNum){ matchBlock->swapBlockCovers(_blockCovers);//got blockCovers - matchBlock->getPackedCover(); matchBlock->packData(); } protected: diff --git a/libHDiffPatch/HDiff/private_diff/suffix_string.cpp b/libHDiffPatch/HDiff/private_diff/suffix_string.cpp index 4c3ffa2d..30d03a85 100644 --- a/libHDiffPatch/HDiff/private_diff/suffix_string.cpp +++ b/libHDiffPatch/HDiff/private_diff/suffix_string.cpp @@ -292,7 +292,7 @@ TInt TSuffixString::lower_bound(const TChar* str,const TChar* str_end)const{ #if (_SSTRING_FAST_MATCH>0) #define kMinStrLen _SSTRING_FAST_MATCH assert((size_t)(str_end-str)>=kMinStrLen); - if (m_isUsedFastMatch&&(!m_fastMatch.isHit(TFastMatchForSString::getHash(str)))) + if (m_isUsedFastMatch&&(!m_fastMatch.isHit(TFastMatchForSString::getHash(str,_SSTRING_FAST_MATCH)))) return -1; #else //assert(str_end-str>=2); @@ -337,7 +337,11 @@ void TSuffixString::clear_cache(){ void TSuffixString::build_cache(size_t threadNum){ clear_cache(); #if (_SSTRING_FAST_MATCH>0) - if (m_isUsedFastMatch) m_fastMatch.buildMatchCache(m_src_begin,m_src_end,threadNum); + #define kFMZoom 4 //ctrl memory size & match speed + if (m_isUsedFastMatch){ + //_out_diff_info(" build cache for sstring fast match ...\n"); + m_fastMatch.buildMatchCache(m_src_begin,m_src_end,threadNum,kFMZoom,_SSTRING_FAST_MATCH); + } #endif const size_t kUsedCacheMinSASize =2*(1<<20); //Enable large cache table only when string is large. if (SASize()>kUsedCacheMinSASize){ @@ -370,14 +374,12 @@ void TSuffixString::build_cache(size_t threadNum){ } -#if (_SSTRING_FAST_MATCH>0) - template static void _filter_insert(TBloomFilter* filter, - const TChar* src_begin,const TChar* src_end){ + const TChar* src_begin,const TChar* src_end,size_t kRollLen){ const TChar* cur = src_begin; - TFastMatchForSString::THash h=TFastMatchForSString::getHash(cur); - cur+=TFastMatchForSString::kFMMinStrSize; + TFastMatchForSString::THash h=TFastMatchForSString::getHash(cur,kRollLen); + cur+=kRollLen; do { #if (_IS_USED_MULTITHREAD) if (isMT) @@ -386,18 +388,18 @@ void TSuffixString::build_cache(size_t threadNum){ #endif filter->insert(h); if (cur=kFMMinStrSize){ - const size_t rollSize=srcSize-(kFMMinStrSize-1); - bf.init(rollSize,kFMZoom); //alloc large memory + if (srcSize>=kRollLen){ + const size_t rollSize=srcSize-(kRollLen-1); + bf.init(rollSize,kBloomZoom); //alloc large memory #if (_IS_USED_MULTITHREAD) const size_t kInsertMinParallelSize=4096; if ((threadNum>1)&&(rollSize>=kInsertMinParallelSize)) { @@ -408,21 +410,20 @@ void TSuffixString::build_cache(size_t threadNum){ const size_t threadCount=threadNum-1; std::vector threads(threadCount); for (size_t i=0;i,&bf,src_begin,src_begin+step+(kFMMinStrSize-1)); - _filter_insert(&bf,src_begin,src_end); + threads[i]=std::thread(_filter_insert,&bf,src_begin,src_begin+step+(kRollLen-1),kRollLen); + _filter_insert(&bf,src_begin,src_end,kRollLen); for (size_t i=0;i(&bf,src_begin,src_end); + _filter_insert(&bf,src_begin,src_end,kRollLen); } }else if ((srcSize>0)||(src_begin!=0)) - bf.init(0,kFMZoom); + bf.init(0,kBloomZoom); else{ bf.clear(); } } -#endif }//namespace hdiff_private diff --git a/libHDiffPatch/HDiff/private_diff/suffix_string.h b/libHDiffPatch/HDiff/private_diff/suffix_string.h index 22c07626..321a5f1a 100644 --- a/libHDiffPatch/HDiff/private_diff/suffix_string.h +++ b/libHDiffPatch/HDiff/private_diff/suffix_string.h @@ -54,25 +54,20 @@ namespace hdiff_private{ # endif #endif -#if (_SSTRING_FAST_MATCH>0) class TFastMatchForSString{ public: - typedef uint32_t THash; + typedef uint64_t THash; typedef unsigned char TChar; - enum { kFMMinStrSize=_SSTRING_FAST_MATCH }; inline TFastMatchForSString(){} inline void clear(){ bf.clear(); } - void buildMatchCache(const TChar* src_begin,const TChar* src_end,size_t threadNum); - - static hpatch_force_inline THash getHash(const TChar* datas) { return fast_adler32_start(datas,kFMMinStrSize); } - static hpatch_force_inline THash rollHash(THash h,const TChar* cur) { return fast_adler32_roll(h,kFMMinStrSize,cur[-kFMMinStrSize],cur[0]); } - + void buildMatchCache(const TChar* src_begin,const TChar* src_end,size_t threadNum,size_t kBloomZoom,size_t kRollLen); + static hpatch_force_inline THash getHash(const TChar* datas,size_t kRollLen) { return fast_adler64_start(datas,kRollLen); } + static hpatch_force_inline THash rollHash(THash h,const TChar* cur,size_t kRollLen) { return fast_adler64_roll(h,kRollLen,*(cur-kRollLen),cur[0]); } hpatch_force_inline bool isHit(THash h) const { return bf.is_hit(h); } private: TBloomFilter bf; }; -#endif class TSuffixString{ public: diff --git a/libHDiffPatch/HDiff/private_diff/window_diff/covers_range.cpp b/libHDiffPatch/HDiff/private_diff/window_diff/covers_range.cpp index 865547b3..74be529a 100644 --- a/libHDiffPatch/HDiff/private_diff/window_diff/covers_range.cpp +++ b/libHDiffPatch/HDiff/private_diff/window_diff/covers_range.cpp @@ -252,6 +252,7 @@ static const TCover* _get_best_clip(const TCover* cover,const TCover* cover_end, hpatch_StreamPos_t gap=_old_gap(c,window); hpatch_StreamPos_t clampedGap=std::min(gap,oldWindowSize); hpatch_StreamPos_t score_factor=4*oldWindowSize-3*clampedGap; + assert(oldWindowSize>0); return (hpatch_uint64_t)c->length*score_factor/(8*oldWindowSize); } } diff --git a/libHDiffPatch/HDiff/private_diff/window_diff/window_matcher.cpp b/libHDiffPatch/HDiff/private_diff/window_diff/window_matcher.cpp index 3aafec5f..37b02f7e 100644 --- a/libHDiffPatch/HDiff/private_diff/window_diff/window_matcher.cpp +++ b/libHDiffPatch/HDiff/private_diff/window_diff/window_matcher.cpp @@ -141,8 +141,8 @@ void extenWindowsForMatch(std::vector& windows,hpatch_StreamPos_ assert(window.newLength>0); if (window.oldLength==0) continue; _extenPos(window.oldPos,window.oldLength,oldWindowSize,0,oldSize,kExtenPosSize); - hpatch_StreamPos_t newLimitPosEnd=(i+1dec_threadNum>1)&&decompressPlugin->is_can_open("lzma2")); + if (isNotDecompressMT) mtsets.decompressDiff_isMT=0; if (mtsets.readOld_isMT){// is can cache all old? size_t workBufCount,objsMemSize,kMinTempCacheSize; hpatchMTSets_t _mtsets=mtsets; _mtsets.readOld_isMT=0; @@ -207,7 +208,7 @@ hpatchMTSets_t _hpatch_getMTSets(hpatch_StreamPos_t newSize,hpatch_StreamPos_t o #endif mtsets=*(hpatchMTSets_t*)disThreads; } - if ((decompressPlugin!=0)&&(mtsets.readDiff_isMT!=0)&&(!mtsets.decompressDiff_isMT)){ + if ((!isNotDecompressMT)&&(mtsets.readDiff_isMT!=0)&&(!mtsets.decompressDiff_isMT)){ mtsets.readDiff_isMT=0; mtsets.decompressDiff_isMT=1; } diff --git a/libHDiffPatch/HPatch/patch.c b/libHDiffPatch/HPatch/patch.c index 11521e7d..c23bdbc8 100644 --- a/libHDiffPatch/HPatch/patch.c +++ b/libHDiffPatch/HPatch/patch.c @@ -2860,6 +2860,7 @@ static const size_t _kWindowCacheCount=_kCacheSgCount; if (!_read_sign_pos_byLastPos(clip,&lastOldPos)) return _hpatch_FALSE; #if (defined __RUN_MEM_SAFE_CHECK) if (len>maxWindowOldSize) return _hpatch_FALSE; + if (len!=(hpatch_StreamPos_t)(hpatch_size_t)len) return _hpatch_FALSE; if (lastOldPos>oldSize) return _hpatch_FALSE; if (lastOldPos+len>oldSize) return _hpatch_FALSE; #endif @@ -2898,7 +2899,7 @@ static hpatch_BOOL _patch_window_diff(const hpatch_TStreamOutput* out_newData,co hpatch_TChecksum* checksumPlugin,hpatch_checksumHandle checksumHandle_old, unsigned char* temp_cache,unsigned char* temp_cache_end, size_t maxThreadNum,hpatchMTSets_t hpatchMTSets){ - hpatch_size_t windowOldBufSize=(hpatch_size_t)diffInfo->maxWindowOldSize; + const hpatch_size_t windowOldBufSize=(hpatch_size_t)diffInfo->maxWindowOldSize; const hpatch_size_t stepMemSize=(hpatch_size_t)diffInfo->maxStepMemSize; const hpatch_StreamPos_t windowCount=diffInfo->windowCount; const hpatch_StreamPos_t metaCount=diffInfo->windowMetaCount; @@ -2933,6 +2934,10 @@ static hpatch_BOOL _patch_window_diff(const hpatch_TStreamOutput* out_newData,co assert(diffStream->read!=0); assert(diffInfo!=0); assert(((checksumHandle_old==0)&&(checksumPlugin==0))||((checksumHandle_old!=0)&&(checksumPlugin!=0))); +#if (defined __RUN_MEM_SAFE_CHECK) + if (diffInfo->maxWindowOldSize!=(hpatch_StreamPos_t)(hpatch_size_t)diffInfo->maxWindowOldSize) return _hpatch_FALSE; + if (diffInfo->maxStepMemSize!=(hpatch_StreamPos_t)(hpatch_size_t)diffInfo->maxStepMemSize) return _hpatch_FALSE; +#endif if (diffInfo->compressedSize==0){ decompressPlugin=0; @@ -2962,6 +2967,9 @@ static hpatch_BOOL _patch_window_diff(const hpatch_TStreamOutput* out_newData,co assert((size_t)(temp_cache_end-temp_cache)>=windowOldBufSize+stepMemSize+hpatch_kStreamCacheSize*kCacheCount); oldBuf=temp_cache; temp_cache+=windowOldBufSize; } +#if (defined __RUN_MEM_SAFE_CHECK) + if ((size_t)(temp_cache_end-temp_cache)uncompressedSize,decompressPlugin,diffStream, diffData_pos,diffData_posEnd)) { result=_hpatch_FALSE; goto _clear; } @@ -2981,7 +2989,7 @@ static hpatch_BOOL _patch_window_diff(const hpatch_TStreamOutput* out_newData,co #endif // Remaining temp_cache for stream caches - cacheSize=(hpatch_size_t)(temp_cache_end-temp_cache-stepMemSize)/kCacheCount; + cacheSize=(hpatch_size_t)((temp_cache_end-temp_cache)-stepMemSize)/kCacheCount; if (cacheSizedecError==hpatch_dec_ok) \ diff --git a/libParallel/parallel_channel.h b/libParallel/parallel_channel.h index d4b0599c..d7dc6b6f 100644 --- a/libParallel/parallel_channel.h +++ b/libParallel/parallel_channel.h @@ -42,7 +42,7 @@ struct CHLocker{ HLocker locker; - inline CHLocker():locker(0) { locker=locker_new(); } + inline explicit CHLocker(bool isAutoNew=true):locker(0) { if (isAutoNew) locker=locker_new(); } inline ~CHLocker() { locker_delete(locker); } };