Work on adding overall checksum support

tools/rsync/algorithm.go

@@ -14,6 +14,8 @@ import (
 	"hash"
 	"io"
 	"os"
+
+	"github.com/zeebo/xxh3"
 )
 
 // If no BlockSize is specified in the rsync instance, this value is used.
@@ -32,6 +34,36 @@ const (
 	OpBlockRange
 )
 
+type xxh3_128 struct {
+	xxh3.Hasher
+}
+
+func (self *xxh3_128) Sum(b []byte) []byte {
+	s := self.Sum128()
+	pos := len(b)
+	if len(b)+16 < cap(b) {
+		var x [16]byte
+		b = append(b, x[:]...)
+	} else {
+		b = b[:len(b)+16]
+	}
+	binary.BigEndian.PutUint64(b[pos:], s.Hi)
+	binary.BigEndian.PutUint64(b[pos+8:], s.Lo)
+	return b
+}
+
+func new_xxh3_64() hash.Hash64 {
+	ans := xxh3.New()
+	ans.Reset()
+	return ans
+}
+
+func new_xxh3_128() hash.Hash {
+	ans := new(xxh3_128)
+	ans.Reset()
+	return ans
+}
+
 // Instruction to mutate target to align to source.
 type Operation struct {
 	Type          OpType
@@ -163,9 +195,11 @@ type rsync struct {
 	BlockSize int
 
 	// This must be non-nil before using any functions
-	hasher             hash.Hash64
+	hasher                  hash.Hash64
-	hasher_constructor func() hash.Hash64
+	hasher_constructor      func() hash.Hash64
-	buffer             []byte
+	checksummer_constructor func() hash.Hash
+	checksummer             hash.Hash
+	buffer                  []byte
 }
 
 func (r *rsync) SetHasher(c func() hash.Hash64) {
@@ -173,6 +207,11 @@ func (r *rsync) SetHasher(c func() hash.Hash64) {
 	r.hasher = c()
 }
 
+func (r *rsync) SetChecksummer(c func() hash.Hash) {
+	r.checksummer_constructor = c
+	r.checksummer = c()
+}
+
 // If the target length is known the number of hashes in the
 // signature can be determined.
 func (r *rsync) BlockHashCount(targetLength int64) (count int64) {

tools/rsync/api.go

@@ -1,15 +1,26 @@
 // License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
 
+// First create a patcher with:
+// p = NewPatcher()
+// Create a signature for the file you want to update using
+// p.CreateSignatureIterator(file_to_update)
+// Now create a Differ with the created signature
+// d = NewDiffer()
+// d.AddSignatureData(signature_data_from_previous_step)
+// Now create a delta based on the signature and the reference file
+// d.CreateDelta(reference_file)
+// Finally, apply this delta using the patcher to produce a file identical to reference_file
+// based ont he delta data and file_to_update
+// p.StartDelta(output_file, file_to_update)
+// p.UpdateDelta(...)
+// p.FinishDelta()
 package rsync
 
 import (
 	"fmt"
-	"hash"
 	"io"
 	"math"
 
-	"github.com/zeebo/xxh3"
-
 	"kitty/tools/utils"
 )
 
@@ -19,10 +30,14 @@ const MaxBlockSize int = 1024 * 1024 // sqrt of 1TB
 
 type StrongHashType uint16
 type WeakHashType uint16
+type ChecksumType uint16
 
 const (
 	XXH3 StrongHashType = iota
 )
+const (
+	XXH3128Sum ChecksumType = iota
+)
 const (
 	Rsync WeakHashType = iota
 )
@@ -33,6 +48,7 @@ type Api struct {
 	rsync     rsync
 	signature []BlockHash
 
+	Checksum_type    ChecksumType
 	Strong_hash_type StrongHashType
 	Weak_hash_type   WeakHashType
 	Grow_buffer_func GrowBufferFunction
@@ -57,13 +73,20 @@ func (self *Api) read_signature_header(data []byte) (consumed int, err error) {
 	if len(data) < 12 {
 		return -1, io.ErrShortBuffer
 	}
-	if version := bin.Uint32(data); version != 0 {
+	if version := bin.Uint16(data); version != 0 {
 		return consumed, fmt.Errorf("Invalid version in signature header: %d", version)
 	}
+	switch csum := ChecksumType(bin.Uint16(data[2:])); csum {
+	case XXH3128Sum:
+		self.Checksum_type = XXH3128Sum
+		self.rsync.SetChecksummer(new_xxh3_128)
+	default:
+		return consumed, fmt.Errorf("Invalid checksum_type in signature header: %d", csum)
+	}
 	switch strong_hash := StrongHashType(bin.Uint16(data[4:])); strong_hash {
 	case XXH3:
 		self.Strong_hash_type = strong_hash
-		self.rsync.SetHasher(func() hash.Hash64 { return xxh3.New() })
+		self.rsync.SetHasher(new_xxh3_64)
 	default:
 		return consumed, fmt.Errorf("Invalid strong_hash in signature header: %d", strong_hash)
 	}
@@ -206,7 +229,8 @@ func (self *Patcher) CreateSignatureIterator(src io.Reader) OutputIterator {
 			var b []byte
 			it = self.rsync.CreateSignatureIterator(src)
 			output, b = ensure_size(output, 12, gbf)
-			bin.PutUint32(b, 0)
+			bin.PutUint16(b, 0)
+			bin.PutUint16(b[2:], uint16(self.Checksum_type))
 			bin.PutUint16(b[4:], uint16(self.Strong_hash_type))
 			bin.PutUint16(b[6:], uint16(self.Weak_hash_type))
 			bin.PutUint32(b[8:], uint32(self.rsync.BlockSize))
@@ -288,7 +312,8 @@ func NewPatcher(expected_input_size int64) (ans *Patcher) {
 	}
 	ans = &Patcher{}
 	ans.rsync.BlockSize = utils.Min(bs, MaxBlockSize)
-	ans.rsync.SetHasher(func() hash.Hash64 { return xxh3.New() })
+	ans.rsync.SetHasher(new_xxh3_64)
+	ans.rsync.SetChecksummer(new_xxh3_128)
 
 	if ans.rsync.HashBlockSize() > 0 && ans.rsync.HashBlockSize() < ans.rsync.BlockSize {
 		ans.rsync.BlockSize = (ans.rsync.BlockSize / ans.rsync.HashBlockSize()) * ans.rsync.HashBlockSize()