Use the append paradigm for the delta iterator as well

tools/rsync/algorithm.go

@@ -44,30 +44,38 @@ type Operation struct {
 
 var bin = binary.LittleEndian
 
-func (self Operation) Serialize() []byte {
+func (self Operation) SerializeSize() int {
-	if self.serialized_repr != nil {
+	switch self.Type {
-		return self.serialized_repr
+	case OpBlock:
-	}
+		return 9
-	var ans []byte
+	case OpBlockRange:
+		return 13
+	case OpHash:
+		return 3 + len(self.Data)
+	case OpData:
+		return 5 + len(self.Data)
+	}
+	return -1
+}
+
+func (self Operation) Serialize(ans []byte) {
+	if self.serialized_repr != nil {
+		copy(ans, self.serialized_repr)
+	}
 	switch self.Type {
 	case OpBlock:
-		ans = make([]byte, 9)
 		bin.PutUint64(ans[1:], self.BlockIndex)
 	case OpBlockRange:
-		ans = make([]byte, 13)
 		bin.PutUint64(ans[1:], self.BlockIndex)
 		bin.PutUint32(ans[9:], uint32(self.BlockIndexEnd-self.BlockIndex))
 	case OpHash:
-		ans = make([]byte, 3+len(self.Data))
 		bin.PutUint16(ans[1:], uint16(len(self.Data)))
 		copy(ans[3:], self.Data)
 	case OpData:
-		ans = make([]byte, 5+len(self.Data))
 		bin.PutUint32(ans[1:], uint32(len(self.Data)))
 		copy(ans[5:], self.Data)
 	}
 	ans[0] = byte(self.Type)
-	return ans
 }
 
 func (self *Operation) Unserialize(data []byte) (n int, err error) {

tools/rsync/api.go

@@ -178,12 +178,12 @@ func write_block_hash(output []byte, bl BlockHash) []byte {
 	return output
 }
 
-// Append the next signature to the provided slice and return the resulting slice. Data is written to the slice iff err == nil.
+// Append the next item to the provided slice and return the resulting slice. Data is written to the slice iff err == nil.
-// When no more signature data is available, err = io.EOF and the unmodified slice is returned.
+// When no more data is available, err = io.EOF and the unmodified slice is returned.
-type SignatureIterator = func([]byte) ([]byte, error)
+type OutputIterator = func([]byte) ([]byte, error)
 
 // Create a signature for the data source in src.
-func (self *Patcher) CreateSignatureIterator(src io.Reader) SignatureIterator {
+func (self *Patcher) CreateSignatureIterator(src io.Reader) OutputIterator {
 	var it func() (BlockHash, error)
 	finished := false
 	return func(output []byte) ([]byte, error) {
@@ -212,24 +212,29 @@ func (self *Patcher) CreateSignatureIterator(src io.Reader) SignatureIterator {
 	}
 }
 
-type DeltaIterator = func() ([]byte, error)
-
 // Create a serialized delta based on the previously loaded signature
-func (self *Differ) CreateDelta(src io.Reader) DeltaIterator {
+func (self *Differ) CreateDelta(src io.Reader) OutputIterator {
 	if err := self.finish_signature_data(); err != nil {
-		return func() ([]byte, error) { return nil, err }
+		return func([]byte) ([]byte, error) { return nil, err }
 	}
 	if self.signature == nil {
-		return func() ([]byte, error) { return nil, fmt.Errorf("Cannot call CreateDelta() before loading a signature") }
+		return func([]byte) ([]byte, error) {
+			return nil, fmt.Errorf("Cannot call CreateDelta() before loading a signature")
+		}
 	}
 	it := self.rsync.CreateDiff(src, self.signature)
-	return func() ([]byte, error) {
+	return func(output []byte) ([]byte, error) {
 		for {
 			op, err := it()
 			if op == nil {
-				return nil, err
+				if err == nil {
+					err = io.EOF
 				}
-			return op.Serialize(), nil
+				return output, err
+			}
+			output, p := ensure_size(output, op.SerializeSize())
+			op.Serialize(p)
+			return output, nil
 		}
 	}
 }

tools/rsync/api_test.go

@@ -96,29 +96,26 @@ func run_roundtrip_test(t *testing.T, src_data, changed []byte, num_of_patches,
 	if err := d.AddSignatureData(signature_of_changed); err != nil {
 		t.Fatal(err)
 	}
-	deltabuf := bytes.Buffer{}
+	deltabuf := make([]byte, 0, 8192)
 	it := d.CreateDelta(bytes.NewBuffer(src_data))
 	for {
-		b, err := it()
+		b, err := it(deltabuf)
-		if b == nil {
 		if err != nil {
-				t.Fatal(err)
+			if err == io.EOF {
-			}
 				break
 			}
-		deltabuf.Write(b)
+			t.Fatal(err)
+		}
+		deltabuf = b
 	}
 	outputbuf := bytes.Buffer{}
 	p.StartDelta(&outputbuf, bytes.NewReader(changed))
-	b := make([]byte, 30*1024)
+	for len(deltabuf) > 0 {
-	for {
+		n := utils.Min(123, len(deltabuf))
-		n, _ := deltabuf.Read(b)
+		if err := p.UpdateDelta(deltabuf[:n]); err != nil {
-		if n <= 0 {
-			break
-		}
-		if err := p.UpdateDelta(b[:n]); err != nil {
 			t.Fatal(err)
 		}
+		deltabuf = deltabuf[n:]
 	}
 	if err := p.FinishDelta(); err != nil {
 		t.Fatal(err)