diff --git a/tools/rsync/algorithm.go b/tools/rsync/algorithm.go index 72c38abc9..90bc0a894 100644 --- a/tools/rsync/algorithm.go +++ b/tools/rsync/algorithm.go @@ -14,6 +14,8 @@ import ( "hash" "io" "os" + + "github.com/zeebo/xxh3" ) // If no BlockSize is specified in the rsync instance, this value is used. @@ -32,6 +34,36 @@ const ( OpBlockRange ) +type xxh3_128 struct { + xxh3.Hasher +} + +func (self *xxh3_128) Sum(b []byte) []byte { + s := self.Sum128() + pos := len(b) + if len(b)+16 < cap(b) { + var x [16]byte + b = append(b, x[:]...) + } else { + b = b[:len(b)+16] + } + binary.BigEndian.PutUint64(b[pos:], s.Hi) + binary.BigEndian.PutUint64(b[pos+8:], s.Lo) + return b +} + +func new_xxh3_64() hash.Hash64 { + ans := xxh3.New() + ans.Reset() + return ans +} + +func new_xxh3_128() hash.Hash { + ans := new(xxh3_128) + ans.Reset() + return ans +} + // Instruction to mutate target to align to source. type Operation struct { Type OpType @@ -163,9 +195,11 @@ type rsync struct { BlockSize int // This must be non-nil before using any functions - hasher hash.Hash64 - hasher_constructor func() hash.Hash64 - buffer []byte + hasher hash.Hash64 + hasher_constructor func() hash.Hash64 + checksummer_constructor func() hash.Hash + checksummer hash.Hash + buffer []byte } func (r *rsync) SetHasher(c func() hash.Hash64) { @@ -173,6 +207,11 @@ func (r *rsync) SetHasher(c func() hash.Hash64) { r.hasher = c() } +func (r *rsync) SetChecksummer(c func() hash.Hash) { + r.checksummer_constructor = c + r.checksummer = c() +} + // If the target length is known the number of hashes in the // signature can be determined. func (r *rsync) BlockHashCount(targetLength int64) (count int64) { diff --git a/tools/rsync/api.go b/tools/rsync/api.go index 8ee541259..86a64d83e 100644 --- a/tools/rsync/api.go +++ b/tools/rsync/api.go @@ -1,15 +1,26 @@ // License: GPLv3 Copyright: 2023, Kovid Goyal, +// First create a patcher with: +// p = NewPatcher() +// Create a signature for the file you want to update using +// p.CreateSignatureIterator(file_to_update) +// Now create a Differ with the created signature +// d = NewDiffer() +// d.AddSignatureData(signature_data_from_previous_step) +// Now create a delta based on the signature and the reference file +// d.CreateDelta(reference_file) +// Finally, apply this delta using the patcher to produce a file identical to reference_file +// based ont he delta data and file_to_update +// p.StartDelta(output_file, file_to_update) +// p.UpdateDelta(...) +// p.FinishDelta() package rsync import ( "fmt" - "hash" "io" "math" - "github.com/zeebo/xxh3" - "kitty/tools/utils" ) @@ -19,10 +30,14 @@ const MaxBlockSize int = 1024 * 1024 // sqrt of 1TB type StrongHashType uint16 type WeakHashType uint16 +type ChecksumType uint16 const ( XXH3 StrongHashType = iota ) +const ( + XXH3128Sum ChecksumType = iota +) const ( Rsync WeakHashType = iota ) @@ -33,6 +48,7 @@ type Api struct { rsync rsync signature []BlockHash + Checksum_type ChecksumType Strong_hash_type StrongHashType Weak_hash_type WeakHashType Grow_buffer_func GrowBufferFunction @@ -57,13 +73,20 @@ func (self *Api) read_signature_header(data []byte) (consumed int, err error) { if len(data) < 12 { return -1, io.ErrShortBuffer } - if version := bin.Uint32(data); version != 0 { + if version := bin.Uint16(data); version != 0 { return consumed, fmt.Errorf("Invalid version in signature header: %d", version) } + switch csum := ChecksumType(bin.Uint16(data[2:])); csum { + case XXH3128Sum: + self.Checksum_type = XXH3128Sum + self.rsync.SetChecksummer(new_xxh3_128) + default: + return consumed, fmt.Errorf("Invalid checksum_type in signature header: %d", csum) + } switch strong_hash := StrongHashType(bin.Uint16(data[4:])); strong_hash { case XXH3: self.Strong_hash_type = strong_hash - self.rsync.SetHasher(func() hash.Hash64 { return xxh3.New() }) + self.rsync.SetHasher(new_xxh3_64) default: return consumed, fmt.Errorf("Invalid strong_hash in signature header: %d", strong_hash) } @@ -206,7 +229,8 @@ func (self *Patcher) CreateSignatureIterator(src io.Reader) OutputIterator { var b []byte it = self.rsync.CreateSignatureIterator(src) output, b = ensure_size(output, 12, gbf) - bin.PutUint32(b, 0) + bin.PutUint16(b, 0) + bin.PutUint16(b[2:], uint16(self.Checksum_type)) bin.PutUint16(b[4:], uint16(self.Strong_hash_type)) bin.PutUint16(b[6:], uint16(self.Weak_hash_type)) bin.PutUint32(b[8:], uint32(self.rsync.BlockSize)) @@ -288,7 +312,8 @@ func NewPatcher(expected_input_size int64) (ans *Patcher) { } ans = &Patcher{} ans.rsync.BlockSize = utils.Min(bs, MaxBlockSize) - ans.rsync.SetHasher(func() hash.Hash64 { return xxh3.New() }) + ans.rsync.SetHasher(new_xxh3_64) + ans.rsync.SetChecksummer(new_xxh3_128) if ans.rsync.HashBlockSize() > 0 && ans.rsync.HashBlockSize() < ans.rsync.BlockSize { ans.rsync.BlockSize = (ans.rsync.BlockSize / ans.rsync.HashBlockSize()) * ans.rsync.HashBlockSize()