Merge pull request #197 from bpot/bp/serialize_stream

Reduce buffering in WriteTo

benchmark_test.go

@@ -469,6 +469,7 @@ func BenchmarkSparseIterateBitset(b *testing.B) {
 }
 
 func BenchmarkSerializationSparse(b *testing.B) {
+	b.ReportAllocs()
 	b.StopTimer()
 	r := rand.New(rand.NewSource(0))
 	s := NewBitmap()
@@ -487,6 +488,7 @@ func BenchmarkSerializationSparse(b *testing.B) {
 }
 
 func BenchmarkSerializationMid(b *testing.B) {
+	b.ReportAllocs()
 	b.StopTimer()
 	r := rand.New(rand.NewSource(0))
 	s := NewBitmap()
@@ -505,6 +507,7 @@ func BenchmarkSerializationMid(b *testing.B) {
 }
 
 func BenchmarkSerializationDense(b *testing.B) {
+	b.ReportAllocs()
 	b.StopTimer()
 	r := rand.New(rand.NewSource(0))
 	s := NewBitmap()

roaringarray.go

@@ -457,8 +457,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 {
 //
 // spec: https://github.com/RoaringBitmap/RoaringFormatSpec
 //
-func (ra *roaringArray) toBytes() ([]byte, error) {
-	stream := &bytes.Buffer{}
+func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
 	hasRun := ra.hasRunCompression()
 	isRunSizeInBytes := 0
 	cookieSize := 8
@@ -523,33 +522,29 @@ func (ra *roaringArray) toBytes() ([]byte, error) {
 		}
 	}
 
-	_, err := stream.Write(buf[:nw])
+	written, err := w.Write(buf[:nw])
 	if err != nil {
-		return nil, err
+		return n, err
 	}
-	for i, c := range ra.containers {
-		_ = i
-		_, err := c.writeTo(stream)
+	n += int64(written)
+
+	for _, c := range ra.containers {
+		written, err := c.writeTo(w)
 		if err != nil {
-			return nil, err
+			return n, err
 		}
+		n += int64(written)
 	}
-	return stream.Bytes(), nil
+	return n, nil
 }
 
 //
 // spec: https://github.com/RoaringBitmap/RoaringFormatSpec
 //
-func (ra *roaringArray) writeTo(out io.Writer) (int64, error) {
-	by, err := ra.toBytes()
-	if err != nil {
-		return 0, err
-	}
-	n, err := out.Write(by)
-	if err == nil && n < len(by) {
-		err = io.ErrShortWrite
-	}
-	return int64(n), err
+func (ra *roaringArray) toBytes() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := ra.writeTo(&buf)
+	return buf.Bytes(), err
 }
 
 func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {