optimize hdiffz -m memory; multi-thread patch base;

.github/workflows/ci.yml

@@ -10,15 +10,16 @@ jobs:
       - uses: actions/checkout@v2
       - name: make_slef
         run: |
-          make CC=gcc CXX=g++ ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 -j
+          make CC=gcc CXX=g++ ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 XXH=0 -j
           make clean
       - name: make_0
         run: |
-          make CC=gcc CXX=g++ DIR_DIFF=0 MT=0 VCD=0 ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 -j
+          make CC=gcc CXX=g++ DIR_DIFF=0 MT=0 VCD=0 ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 XXH=0 -j
           make clean
       - name: make_all
         run: |
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
+          git clone --depth=1 https://github.com/sisong/xxHash.git ../xxHash
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
@@ -37,11 +38,12 @@ jobs:
       - uses: actions/checkout@v2
       - name: make_slefByClang
         run: |
-          make CL=1 ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 -j
+          make CL=1 ZLIB=0 BSD=0 BZIP2=0 LDEF=0 LZMA=0 ZSTD=0 MD5=0 XXH=0 -j
           make clean
       - name: make_allByClang
         run: |
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
+          git clone --depth=1 https://github.com/sisong/xxHash.git ../xxHash
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2
@@ -58,6 +60,7 @@ jobs:
       - name: initAndClone
         run: |
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
+          git clone --depth=1 https://github.com/sisong/xxHash.git ../xxHash
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           git clone --depth=1 https://github.com/sisong/zlib.git ../zlib
@@ -88,7 +91,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: nttld/[email protected]
         with:
-          ndk-version: r16b
+          ndk-version: r23c
       - name: buildByAndroidNDK
         run: |
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
@@ -103,7 +106,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: nttld/[email protected]
         with:
-          ndk-version: r16b
+          ndk-version: r23c
       - name: buildByAndroidNDK
         run: |
           curl -O http://launchpadlibrarian.net/648013231/libtinfo5_6.4-2_amd64.deb
@@ -126,6 +129,7 @@ jobs:
       - name: initAndClone
         run: |
           git clone --depth=1 https://github.com/sisong/libmd5.git ../libmd5
+          git clone --depth=1 https://github.com/sisong/xxHash.git ../xxHash
           git clone --depth=1 https://github.com/sisong/lzma.git ../lzma
           git clone --depth=1 https://github.com/sisong/zstd.git ../zstd
           git clone --depth=1 https://github.com/sisong/bzip2.git ../bzip2

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 full changelog at: https://github.com/sisong/HDiffPatch/commits   
 
+## [v4.12.0](https://github.com/sisong/HDiffPatch/tree/v4.12.0) - 2025-09-18
+### Added
+* optimize `$hdiffz -m` required memory size when newData similar to oldData;
+* patch_single_stream support pre-read some old data for optimize disk read, when patch with large cache memory;
+* now, opened xxh3 & xxh128 ChecksumPlugin when dir-diff, checksum fast; (NOTE: need v4.12 patcher);
+* cmdline hpatchz support show progress;
+* cmdline hpatchz support multi-thread for single compressed diff (create by $hdiffz -SD);   
+  multi-thread used for I/O & decompress, maybe help when patch;
+### Changed
+* recode C++ multi-thread code to C code base, so remove _IS_USED_CPP11THREAD define.
+* update min NDK version from r16b to r23c; ( android 14 API to 16; remove .so for armeabi )
+* Android .so makefile build support directory patch (default closed, need set DIR=1);
+* when hdiffz test diffFile, all check-patcher run with stream mode(slower than run in memory);
+
 ## [v4.11.1](https://github.com/sisong/HDiffPatch/tree/v4.11.1) - 2025-08-18
 ### Fixed
 * fix a bug when run dir_diff by multi-thread parallel on linux;
@@ -64,6 +78,8 @@ if diffFile created by empty oldPath, then extract with default option `$selfExt
 ### Added
 * add libhsync for diff&patch by sync; see demo [hsynz] (like [zsync])
 * add function create_sync_data(),create_dir_sync_data(),sync_patch(),sync_patch_...(),sync_local_diff(),sync_local_diff_...(),sync_local_patch(),sync_local_patch_...()
+* add xxh3,xxh128,sha256,sha512 ChecksumPlugin; default closed;
+
 
 ## [v4.5.2](https://github.com/sisong/HDiffPatch/tree/v4.5.2) - 2022-12-25
 ### Fixed
@@ -105,6 +121,10 @@ if diffFile created by empty oldPath, then extract with default option `$selfExt
 * add function create_bsdiff() & bspatch_with_cache().
 * add function create_single_compressed_diff_block() & create_compressed_diff_block() & create_bsdiff_block().
 
+## [v4.0.4](https://github.com/sisong/HDiffPatch/tree/v4.0.4) - 2021-06-28
+### Added
+* add blake3 ChecksumPlugin; default closed;
+
 ## [v4.0.0](https://github.com/sisong/HDiffPatch/tree/v4.0.0) - 2021-06-14
 ### Added
 * cmdline add option "-SD", to create single compressed diffData, for optimize decompress buffer when patch, and support step by step patching when step by step downloading; it's better for IoT!  NOTE: old patcher can't work with this new format diffData. 
@@ -154,7 +174,7 @@ if diffFile created by empty oldPath, then extract with default option `$selfExt
 ## [v3.0.0](https://github.com/sisong/HDiffPatch/tree/v3.0.0) - 2019-03-01
 ### Added
 * hdiffz,hpatchz command line support diff&patch between directories(folder);
-* support checksum plugin: crc32ChecksumPlugin,adler32ChecksumPlugin,adler64ChecksumPlugin,fadler32ChecksumPlugin,fadler64ChecksumPlugin,fadler128ChecksumPlugin,md5ChecksumPlugin ;
+* add crc32,adler32,adler64,fadler32,fadler64,fadler128,md5 ChecksumPlugin;
 * lzmaCompressPlugin support parallel compress;
 * add parallel compress plugin: pzlibCompressPlugin,pbz2CompressPlugin,lzma2CompressPlugin
 * command line support SFX(self extract archive);

Makefile

@@ -19,6 +19,7 @@ USE_CRC_EMU := 0
 # 0: not need zstd;  1: compile zstd source code;  2: used -lzstd to link zstd lib;
 ZSTD     := 1
 MD5      := 1
+XXH      := 1
 # used clang?
 CL  	 := 0
 # build with -m32?
@@ -66,6 +67,16 @@ HPATCH_OBJ := \
     libHDiffPatch/HPatch/patch.o \
     file_for_patch.o
 
+ifeq ($(MT),0)
+else
+  HPATCH_OBJ+=libHDiffPatch/HPatch/hpatch_mt/_hcache_old_mt.o \
+              libHDiffPatch/HPatch/hpatch_mt/_hinput_mt.o \
+              libHDiffPatch/HPatch/hpatch_mt/_houtput_mt.o \
+              libHDiffPatch/HPatch/hpatch_mt/_hpatch_mt.o \
+              libHDiffPatch/HPatch/hpatch_mt/hpatch_mt.o \
+              libParallel/parallel_import_c.o
+endif
+
 ifeq ($(DIR_DIFF),0)
 else
   HPATCH_OBJ += \
@@ -92,6 +103,8 @@ else
   endif
 endif
 
+XXH_PATH := ../xxHash
+
 LZMA_PATH := ../lzma/C
 ifeq ($(LZMA),1)
   # https://www.7-zip.org  https://github.com/sisong/lzma
@@ -236,13 +249,12 @@ endif
 ifeq ($(MT),0)
 else
   HDIFF_OBJ += \
-    libParallel/parallel_import.o \
     libParallel/parallel_channel.o \
     compress_parallel.o
 endif
 
 UTEST_OBJ := \
-    libHDiffPatch/HDiff/match_inplace.o \
+    libHDiffPatch/HDiff/private_diff/match_inplace.o \
     libhsync/sync_client/dir_sync_client.o \
     libhsync/sync_client/match_in_old.o \
     libhsync/sync_client/sync_client.o \
@@ -259,6 +271,9 @@ DEF_FLAGS := \
     -D_IS_NEED_DEFAULT_CompressPlugin=0 \
     -D_IS_NEED_ALL_ChecksumPlugin=0 \
     -D_IS_NEED_DEFAULT_ChecksumPlugin=0 
+ifeq ($(OS),Windows_NT) # mingw?
+  DEF_FLAGS += -D_IS_USED_WIN32_UTF8_WAPI=1
+endif
 ifeq ($(ATOMIC_U64),0)
   DEF_FLAGS += -D_IS_NO_ATOMIC_U64=1
 endif
@@ -317,6 +332,10 @@ else
   else
     DEF_FLAGS += -D_ChecksumPlugin_md5 -I$(MD5_PATH)
   endif
+  ifeq ($(XXH),0)
+  else
+    DEF_FLAGS += -D_ChecksumPlugin_xxh3 -D_ChecksumPlugin_xxh128 -I$(XXH_PATH)
+  endif
 endif
 ifeq ($(BSD),0)
 	DEF_FLAGS += -D_IS_NEED_BSDIFF=0
@@ -354,6 +373,7 @@ else
   ifeq ($(ZSTD),1)
     DEF_FLAGS += -DZSTD_HAVE_WEAK_SYMBOLS=0 -DZSTD_TRACE=0 -DZSTD_DISABLE_ASM=1 -DZSTDLIB_HIDDEN= \
                  -DZSTDLIB_VISIBLE= -DZDICTLIB_VISIBLE= -DZSTDERRORLIB_VISIBLE= \
+                 -DZSTD_LIB_DEPRECATED=0 -DZSTD_STRIP_ERROR_STRINGS=1 \
 	               -I$(ZSTD_PATH) -I$(ZSTD_PATH)/common -I$(ZSTD_PATH)/compress -I$(ZSTD_PATH)/decompress
 	endif
 endif
@@ -365,8 +385,7 @@ ifeq ($(MT),0)
 else
   DEF_FLAGS += \
     -DZSTD_MULTITHREAD=1 \
-    -D_IS_USED_MULTITHREAD=1 \
-    -D_IS_USED_PTHREAD=1
+    -D_IS_USED_MULTITHREAD=1
 endif
 
 PATCH_LINK := 

README.md

@@ -1,5 +1,5 @@
 # [HDiffPatch]
-[![release](https://img.shields.io/badge/release-v4.11.1-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v4.12.0-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues)   
@@ -56,15 +56,16 @@ apply the delta:
 `$ cd <dir>/HDiffPatch`   
 ### Linux or MacOS X ###
 Try:   
-`$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0`   
+`$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0 XXH=0`   
 bzip2 : if the build fails with `fatal error: bzlib.h: No such file or directory`, use your system's package manager to install the libbz2 package and try again; or download & make with libbz2 source code:
 ```
 $ git clone https://github.com/sisong/bzip2.git ../bzip2
-$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0 BZIP2=1
+$ make LDEF=0 LZMA=0 ZSTD=0 MD5=0 XXH=0 BZIP2=1
 ```
-if need lzma zstd & md5 ... default support, Try:
+if need lzma zstd & md5 xxh... default support, Try:
 ```
 $ git clone https://github.com/sisong/libmd5.git ../libmd5
+$ git clone https://github.com/sisong/xxHash.git ../xxHash
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
 $ git clone https://github.com/sisong/zlib.git ../zlib
@@ -77,6 +78,7 @@ Tip: You can use `$ make -j` to compile in parallel.
 Before you build `builds/vc/HDiffPatch.sln` by [`Visual Studio`](https://visualstudio.microsoft.com), first get the libraries into sibling folders, like so: 
 ```
 $ git clone https://github.com/sisong/libmd5.git ../libmd5
+$ git clone https://github.com/sisong/xxHash.git ../xxHash
 $ git clone https://github.com/sisong/lzma.git ../lzma
 $ git clone https://github.com/sisong/zstd.git ../zstd
 $ git clone https://github.com/sisong/zlib.git   ../zlib
@@ -116,9 +118,9 @@ options:
       matchBlockSize>=4, DEFAULT -s-64, recommended 16,32,48,1k,64k,1m etc...
   -block-fastMatchBlockSize
       must run with -m;
-      set block match befor slow byte-by-byte match, DEFAULT -block-4k;
+      set block match befor slow byte-by-byte match, DEFAULT -block-1k;
       if set -block-0, means don't use block match;
-      fastMatchBlockSize>=4, recommended 256,1k,64k,1m etc...
+      fastMatchBlockSize>=4, recommended 128,4k,64k, etc...
       if newData similar to oldData then diff speed++ & diff memory--,
       but small possibility outDiffFile's size+
   -cache
@@ -129,6 +131,7 @@ options:
   -SD[-stepSize]
       create single compressed diffData, only need one decompress buffer
       when patch, and support step by step patching when step by step downloading!
+        and supports multi-thread patching!
       stepSize>=(1024*4), DEFAULT -SD-256k, recommended 64k,2m etc...
   -BSD
       create diffFile compatible with bsdiff4, unsupport input directory(folder).
@@ -146,11 +149,11 @@ options:
       if parallelThreadNumber>1 then open multi-thread Parallel mode;
       DEFAULT -p-4; requires more memory!
   -p-search-searchThreadNumber
-      must run with -s[-matchBlockSize];
       DEFAULT searchThreadNumber same as parallelThreadNumber;
-      but multi-thread search need frequent random disk reads when matchBlockSize
-      is small, so some times multi-thread maybe much slower than single-thread!
-      if (searchThreadNumber<=1) then to close multi-thread search mode.
+      old file on HDD hard drives WARNING: multi-thread search need frequent random
+        disk reads when -s-matchBlockSize or -block-fastMatchBlockSize(run with -m),
+        causes slowdown; at this time, need to close(searchThreadNumber<=1) multi-thread
+        search mode or reduce the number of searchThreadNumber!
   -c-compressType[-compressLevel]
       set outDiffFile Compress type, DEFAULT uncompress;
       for resave diffFile,recompress diffFile to outDiffFile by new set;
@@ -183,9 +186,11 @@ options:
         -C-crc32
         -C-fadler64             DEFAULT
         -C-md5
+        -C-xxh3                 (need v4.12 patcher)
+        -C-xxh128               recommended (need v4.12 patcher)
   -n-maxOpenFileNumber
       limit Number of open files at same time when stream directory diff;
-      maxOpenFileNumber>=8, DEFAULT -n-48, the best limit value by different
+      maxOpenFileNumber>=16, DEFAULT -n-48, the best limit value by different
         operating system.
   -g#ignorePath[#ignorePath#...]
       set iGnore path list when Directory Diff; ignore path list such as:
@@ -238,8 +243,8 @@ extract SFX: **selfExtractArchive**  (same as: $selfExtractArchive -f {""|".\"}
   if oldPath is empty input parameter ""
 options:
   -s[-cacheSize]
-      DEFAULT -s-4m; oldPath loaded as Stream;
-      cacheSize can like 262144 or 256k or 512m or 2g etc....
+      DEFAULT -s-8m; oldPath loaded as Stream;
+      cacheSize can like 262144 or 256k or 64m or 512m etc....
       requires (cacheSize + 4*decompress buffer size)+O(1) bytes of memory.
       if diffFile is single compressed diffData(created by hdiffz -SD-stepSize), then requires
         (cacheSize+ stepSize + 1*decompress buffer size)+O(1) bytes of memory;
@@ -255,6 +260,10 @@ options:
         (oldFileSize + 3*decompress buffer size)+O(1) bytes of memory.
       if diffFile is VCDIFF(created by hdiffz -VCD,xdelta3,open-vcdiff), then requires
         (sourceWindowSize+targetWindowSize + 3*decompress buffer size)+O(1) bytes of memory.
+  -p-parallelThreadNumber
+      if parallelThreadNumber>1 then open multi-thread Parallel mode;
+      now only support single compressed diffData(created by hdiffz -SD-stepSize);
+      can set 1..5, DEFAULT -p-1!
   -C-checksumSets
       set Checksum data for directory patch, DEFAULT -C-new-copy;
       checksumSets support (can choose multiple):