From 80b4cf24d4963910cd72df2c49b7647c560d18dd Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 17 Sep 2024 15:28:57 +1000 Subject: [PATCH 1/2] Test FromUnsafeBytes return count --- roaring64/serialization_test.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/roaring64/serialization_test.go b/roaring64/serialization_test.go index 31d0e514..5660133f 100644 --- a/roaring64/serialization_test.go +++ b/roaring64/serialization_test.go @@ -17,6 +17,18 @@ import ( "github.com/stretchr/testify/require" ) +func fromUnsafeBytesChecked(t *testing.T, unsafeBm *Bitmap, b []byte) { + var r bytes.Reader + r.Reset(b) + var safeBm Bitmap + safeCount, err := safeBm.ReadFrom(&r) + unsafeCount, err := unsafeBm.FromUnsafeBytes(b) + require.NoError(t, err) + require.NoError(t, err) + assert.EqualValues(t, safeCount, unsafeCount) + assert.True(t, safeBm.Equals(unsafeBm)) +} + func TestSerializationOfEmptyBitmap(t *testing.T) { rb := NewBitmap() @@ -34,8 +46,7 @@ func TestSerializationOfEmptyBitmap(t *testing.T) { assert.True(t, rb.Equals(newrb)) newrb2 := NewBitmap() - _, err = newrb2.FromUnsafeBytes(data) - require.NoError(t, err) + fromUnsafeBytesChecked(t, newrb2, data) assert.True(t, rb.Equals(newrb2)) } @@ -70,8 +81,7 @@ func TestSerializationBasic037(t *testing.T) { assert.True(t, rb.Equals(newrb)) newrb2 := NewBitmap() - _, err = newrb2.FromUnsafeBytes(data) - require.NoError(t, err) + fromUnsafeBytesChecked(t, newrb2, data) assert.True(t, rb.Equals(newrb2)) } @@ -113,8 +123,7 @@ func TestSerializationToFile038(t *testing.T) { assert.True(t, rb.Equals(newrb)) newrb2 := NewBitmap() - _, err = newrb2.FromUnsafeBytes(buf.Bytes()) - require.NoError(t, err) + fromUnsafeBytesChecked(t, newrb2, buf.Bytes()) assert.True(t, rb.Equals(newrb2)) } @@ -136,8 +145,7 @@ func TestSerializationBasic2_041(t *testing.T) { assert.True(t, rb.Equals(newrb)) newrb2 := NewBitmap() - _, err = newrb2.FromUnsafeBytes(data) - require.NoError(t, err) + fromUnsafeBytesChecked(t, newrb2, data) assert.True(t, rb.Equals(newrb2)) } @@ -162,8 +170,7 @@ func TestSerializationBasic3_042(t *testing.T) { assert.True(t, newrb.Equals(rb)) newrb2 := NewBitmap() - _, err = newrb2.FromUnsafeBytes(data) - require.NoError(t, err) + fromUnsafeBytesChecked(t, newrb2, data) assert.True(t, rb.Equals(newrb2)) } From 07dc72f5900e1b90de8355a977ee0abfce08e22f Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 17 Sep 2024 09:57:27 -0400 Subject: [PATCH 2/2] fix returned value for roaring64 read from unsafe bytes --- roaring64/roaring64.go | 10 ++++------ roaring64/roaring64cow_test.go | 1 - roaring64/serialization_test.go | 4 ++-- roaring_test.go | 3 --- roaringcow_test.go | 1 - 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/roaring64/roaring64.go b/roaring64/roaring64.go index 712f3ea4..ebea5ffc 100644 --- a/roaring64/roaring64.go +++ b/roaring64/roaring64.go @@ -87,11 +87,10 @@ func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) { func (rb *Bitmap) FromUnsafeBytes(data []byte) (p int64, err error) { stream := internal.NewByteBuffer(data) sizeBuf := make([]byte, 8) - n, err := stream.Read(sizeBuf) + _, err = stream.Read(sizeBuf) if err != nil { return 0, err } - p += int64(n) size := binary.LittleEndian.Uint64(sizeBuf) rb.highlowcontainer.resize(0) @@ -115,17 +114,16 @@ func (rb *Bitmap) FromUnsafeBytes(data []byte) (p int64, err error) { if err != nil { return 0, fmt.Errorf("error in bitmap.UnsafeFromBytes: could not read key #%d: %w", i, err) } - p += 4 rb.highlowcontainer.keys[i] = binary.LittleEndian.Uint32(keyBuf) rb.highlowcontainer.containers[i] = roaring.NewBitmap() n, err := rb.highlowcontainer.containers[i].ReadFrom(stream) + if n == 0 || err != nil { return int64(n), fmt.Errorf("Could not deserialize bitmap for key #%d: %s", i, err) } - p += int64(n) } - return p, nil + return stream.GetReadBytes(), nil } // ReadFrom reads a serialized version of this bitmap from stream. @@ -167,12 +165,12 @@ func (rb *Bitmap) ReadFrom(stream io.Reader) (p int64, err error) { rb.highlowcontainer.keys[i] = binary.LittleEndian.Uint32(keyBuf) rb.highlowcontainer.containers[i] = roaring.NewBitmap() n, err := rb.highlowcontainer.containers[i].ReadFrom(stream) + if n == 0 || err != nil { return int64(n), fmt.Errorf("Could not deserialize bitmap for key #%d: %s", i, err) } p += int64(n) } - return p, nil } diff --git a/roaring64/roaring64cow_test.go b/roaring64/roaring64cow_test.go index b3b7bee9..7466e56c 100644 --- a/roaring64/roaring64cow_test.go +++ b/roaring64/roaring64cow_test.go @@ -1481,7 +1481,6 @@ func TestBigRandomCOW(t *testing.T) { } func rTestCOW(t *testing.T, N int) { - log.Println("rtest N=", N) for gap := 1; gap <= 65536; gap *= 2 { bs1 := bitset.New(0) rb1 := NewBitmap() diff --git a/roaring64/serialization_test.go b/roaring64/serialization_test.go index 5660133f..7655d8b8 100644 --- a/roaring64/serialization_test.go +++ b/roaring64/serialization_test.go @@ -20,13 +20,13 @@ import ( func fromUnsafeBytesChecked(t *testing.T, unsafeBm *Bitmap, b []byte) { var r bytes.Reader r.Reset(b) - var safeBm Bitmap + safeBm := NewBitmap() safeCount, err := safeBm.ReadFrom(&r) unsafeCount, err := unsafeBm.FromUnsafeBytes(b) require.NoError(t, err) require.NoError(t, err) - assert.EqualValues(t, safeCount, unsafeCount) assert.True(t, safeBm.Equals(unsafeBm)) + assert.EqualValues(t, safeCount, unsafeCount) } func TestSerializationOfEmptyBitmap(t *testing.T) { diff --git a/roaring_test.go b/roaring_test.go index cd299bc1..520da6ab 100644 --- a/roaring_test.go +++ b/roaring_test.go @@ -95,8 +95,6 @@ func checkValidity(t *testing.T, rb *Bitmap) { } func hashTest(t *testing.T, N uint64) { - t.Log("rtest N=", N) - hashes := map[uint64]struct{}{} count := 0 @@ -1944,7 +1942,6 @@ func TestHash(t *testing.T) { } func rTest(t *testing.T, N int) { - t.Log("rtest N=", N) for gap := 1; gap <= 65536; gap *= 2 { bs1 := bitset.New(0) rb1 := NewBitmap() diff --git a/roaringcow_test.go b/roaringcow_test.go index 94d07acb..c04acae4 100644 --- a/roaringcow_test.go +++ b/roaringcow_test.go @@ -1533,7 +1533,6 @@ func TestBigRandomCOW(t *testing.T) { } func rTestCOW(t *testing.T, N int) { - t.Log("rtest N=", N) for gap := 1; gap <= 65536; gap *= 2 { bs1 := bitset.New(0) rb1 := NewBitmap()