From 6d614a98ac4b9f7f8bb802d66281f0b9301615ee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Jul 2023 17:09:58 +0530 Subject: [PATCH] Get the test to pass --- tools/rsync/algorithm.go | 20 +++++++++-------- tools/rsync/api.go | 2 +- tools/rsync/api_test.go | 48 ++++++++++++++++++++++++++++++---------- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/tools/rsync/algorithm.go b/tools/rsync/algorithm.go index 51b856e2a..916b3ad45 100644 --- a/tools/rsync/algorithm.go +++ b/tools/rsync/algorithm.go @@ -92,7 +92,7 @@ func (self BlockHash) Serialize() []byte { return ans } -func (self BlockHash) Unserialize(data []byte, hash_size int) (err error) { +func (self *BlockHash) Unserialize(data []byte, hash_size int) (err error) { if len(data) < 12+hash_size { return fmt.Errorf("record too small to be a BlockHash: %d < %d", len(data), 12+hash_size) } @@ -174,10 +174,7 @@ func (r *RSync) ApplyDelta(alignedTarget io.Writer, target io.ReadSeeker, op Ope var n int var block []byte - minBufferSize := r.BlockSize - if len(r.buffer) < minBufferSize { - r.buffer = make([]byte, minBufferSize) - } + r.set_buffer_to_size(r.BlockSize) buffer := r.buffer writeBlock := func(op Operation) error { @@ -227,16 +224,21 @@ func (r *RSync) ApplyDelta(alignedTarget io.Writer, target io.ReadSeeker, op Ope return nil } +func (r *RSync) set_buffer_to_size(sz int) { + if cap(r.buffer) < sz { + r.buffer = make([]byte, sz) + } else { + r.buffer = r.buffer[:sz] + } +} + // Create the operation list to mutate the target signature into the source. // Any data operation from the OperationWriter must have the data copied out // within the span of the function; the data buffer underlying the operation // data is reused. The sourceSum create a complete hash sum of the source if // present. func (r *RSync) CreateDelta(source io.Reader, signature []BlockHash, ops OperationWriter) (err error) { - minBufferSize := (r.BlockSize * 2) + (r.MaxDataOp) - if len(r.buffer) < minBufferSize { - r.buffer = make([]byte, minBufferSize) - } + r.set_buffer_to_size((r.BlockSize * 2) + (r.MaxDataOp)) buffer := r.buffer // A single β hashes may correlate with a many unique hashes. diff --git a/tools/rsync/api.go b/tools/rsync/api.go index 7dbf432b8..05fbed7b5 100644 --- a/tools/rsync/api.go +++ b/tools/rsync/api.go @@ -78,7 +78,7 @@ func (self *Api) read_signature_header(data []byte) (consumed int, err error) { } self.rsync.BlockSize = block_size self.rsync.MaxDataOp = 10 * block_size - self.signature = make([]BlockHash, 0, 256) + self.signature = make([]BlockHash, 0, 1024) return } diff --git a/tools/rsync/api_test.go b/tools/rsync/api_test.go index 0848252e0..54f43eb60 100644 --- a/tools/rsync/api_test.go +++ b/tools/rsync/api_test.go @@ -9,10 +9,12 @@ import ( "kitty/tools/utils/random" "testing" + "github.com/google/go-cmp/cmp" "golang.org/x/exp/slices" ) var _ = fmt.Print +var _ = cmp.Diff func TestRsyncRoundtrip(t *testing.T) { src_data := make([]byte, 4*1024*1024) @@ -27,13 +29,45 @@ func TestRsyncRoundtrip(t *testing.T) { } return } + test_equal := func(src_data, output []byte) { + if !bytes.Equal(src_data, output) { + first_diff := utils.Min(len(src_data), len(output)) + for i := 0; i < first_diff; i++ { + if src_data[i] != output[i] { + first_diff = i + break + } + } + t.Fatalf("Patching failed: %d extra_bytes first different byte at: %d", len(output)-len(src_data), first_diff) + } + } changed := slices.Clone(src_data) for i := 0; i < 8; i++ { random_patch(changed) } + // first try just the engine without serialization p := NewPatcher(int64(len(src_data))) + signature := make([]BlockHash, 0, 128) + p.rsync.CreateSignature(bytes.NewReader(changed), func(s BlockHash) error { + signature = append(signature, s) + return nil + }) + delta_ops := make([]Operation, 0, 128) + p.rsync.CreateDelta(bytes.NewReader(src_data), signature, func(op Operation) error { + op.Data = slices.Clone(op.Data) + delta_ops = append(delta_ops, op) + return nil + }) + outputbuf := bytes.Buffer{} + for _, op := range delta_ops { + p.rsync.ApplyDelta(&outputbuf, bytes.NewReader(src_data), op) + } + test_equal(src_data, outputbuf.Bytes()) + + // Now try with serialization + p = NewPatcher(int64(len(src_data))) sigbuf := bytes.Buffer{} if err := p.CreateSignature(bytes.NewReader(changed), func(p []byte) error { _, err := sigbuf.Write(p); return err }); err != nil { t.Fatal(err) @@ -46,7 +80,7 @@ func TestRsyncRoundtrip(t *testing.T) { if err := d.CreateDelta(bytes.NewReader(src_data), func(b []byte) error { _, err := deltabuf.Write(b); return err }); err != nil { t.Fatal(err) } - outputbuf := bytes.Buffer{} + outputbuf = bytes.Buffer{} p.StartDelta(&outputbuf, bytes.NewReader(src_data)) b := make([]byte, 30*1024) for { @@ -62,16 +96,6 @@ func TestRsyncRoundtrip(t *testing.T) { t.Fatal(err) } - output := outputbuf.Bytes() - if !bytes.Equal(src_data, output) { - first_diff := utils.Min(len(src_data), len(output)) - for i := 0; i < first_diff; i++ { - if src_data[i] != output[i] { - first_diff = i - break - } - } - t.Fatalf("Patching failed: %d extra_bytes first different byte at: %d", len(outputbuf.Bytes())-len(src_data), first_diff) - } + test_equal(src_data, outputbuf.Bytes()) }