feat: stamp hash migration perf (ethersphere#4732)

pkg/storer/internal/reserve/olditems.go

@@ -0,0 +1,180 @@
+// Copyright 2024 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package reserve
+
+import (
+	"encoding/binary"
+	"path"
+
+	"github.com/ethersphere/bee/v2/pkg/storage"
+	"github.com/ethersphere/bee/v2/pkg/swarm"
+)
+
+// BatchRadiusItemV1 allows iteration of the chunks with respect to bin and batchID.
+// Used for batch evictions of certain bins.
+type BatchRadiusItemV1 struct {
+	Bin     uint8
+	BatchID []byte
+	Address swarm.Address
+	BinID   uint64
+}
+
+func (b *BatchRadiusItemV1) Namespace() string {
+	return "batchRadius"
+}
+
+func (b *BatchRadiusItemV1) ID() string {
+	return string(b.BatchID) + string(b.Bin) + b.Address.ByteString()
+}
+
+func (b *BatchRadiusItemV1) String() string {
+	return path.Join(b.Namespace(), b.ID())
+}
+
+func (b *BatchRadiusItemV1) Clone() storage.Item {
+	if b == nil {
+		return nil
+	}
+	return &BatchRadiusItemV1{
+		Bin:     b.Bin,
+		BatchID: copyBytes(b.BatchID),
+		Address: b.Address.Clone(),
+		BinID:   b.BinID,
+	}
+}
+
+const batchRadiusItemSizeV1 = 1 + swarm.HashSize + swarm.HashSize + 8
+
+func (b *BatchRadiusItemV1) Marshal() ([]byte, error) {
+
+	if b.Address.IsZero() {
+		return nil, errMarshalInvalidAddress
+	}
+
+	buf := make([]byte, batchRadiusItemSizeV1)
+
+	i := 0
+
+	buf[i] = b.Bin
+	i += 1
+
+	copy(buf[i:i+swarm.HashSize], b.BatchID)
+	i += swarm.HashSize
+
+	copy(buf[i:i+swarm.HashSize], b.Address.Bytes())
+	i += swarm.HashSize
+
+	binary.BigEndian.PutUint64(buf[i:i+8], b.BinID)
+
+	return buf, nil
+}
+
+func (b *BatchRadiusItemV1) Unmarshal(buf []byte) error {
+
+	if len(buf) != batchRadiusItemSizeV1 {
+		return errUnmarshalInvalidSize
+	}
+
+	i := 0
+	b.Bin = buf[i]
+	i += 1
+
+	b.BatchID = copyBytes(buf[i : i+swarm.HashSize])
+	i += swarm.HashSize
+
+	b.Address = swarm.NewAddress(buf[i : i+swarm.HashSize]).Clone()
+	i += swarm.HashSize
+
+	b.BinID = binary.BigEndian.Uint64(buf[i : i+8])
+
+	return nil
+}
+
+// ChunkBinItemV1 allows for iterating on ranges of bin and binIDs for chunks.
+// BinIDs come in handy when syncing the reserve contents with other peers.
+type ChunkBinItemV1 struct {
+	Bin       uint8
+	BinID     uint64
+	Address   swarm.Address
+	BatchID   []byte
+	ChunkType swarm.ChunkType
+}
+
+func (c *ChunkBinItemV1) Namespace() string {
+	return "chunkBin"
+}
+
+func (c *ChunkBinItemV1) ID() string {
+	return binIDToString(c.Bin, c.BinID)
+}
+
+func (c *ChunkBinItemV1) String() string {
+	return path.Join(c.Namespace(), c.ID())
+}
+
+func (c *ChunkBinItemV1) Clone() storage.Item {
+	if c == nil {
+		return nil
+	}
+	return &ChunkBinItemV1{
+		Bin:       c.Bin,
+		BinID:     c.BinID,
+		Address:   c.Address.Clone(),
+		BatchID:   copyBytes(c.BatchID),
+		ChunkType: c.ChunkType,
+	}
+}
+
+const chunkBinItemSizeV1 = 1 + 8 + swarm.HashSize + swarm.HashSize + 1
+
+func (c *ChunkBinItemV1) Marshal() ([]byte, error) {
+
+	if c.Address.IsZero() {
+		return nil, errMarshalInvalidAddress
+	}
+
+	buf := make([]byte, chunkBinItemSizeV1)
+	i := 0
+
+	buf[i] = c.Bin
+	i += 1
+
+	binary.BigEndian.PutUint64(buf[i:i+8], c.BinID)
+	i += 8
+
+	copy(buf[i:i+swarm.HashSize], c.Address.Bytes())
+	i += swarm.HashSize
+
+	copy(buf[i:i+swarm.HashSize], c.BatchID)
+	i += swarm.HashSize
+
+	buf[i] = uint8(c.ChunkType)
+
+	return buf, nil
+}
+
+func (c *ChunkBinItemV1) Unmarshal(buf []byte) error {
+
+	if len(buf) != chunkBinItemSizeV1 {
+		return errUnmarshalInvalidSize
+	}
+
+	i := 0
+	c.Bin = buf[i]
+	i += 1
+
+	c.BinID = binary.BigEndian.Uint64(buf[i : i+8])
+	i += 8
+
+	c.Address = swarm.NewAddress(buf[i : i+swarm.HashSize]).Clone()
+	i += swarm.HashSize
+
+	c.BatchID = copyBytes(buf[i : i+swarm.HashSize])
+	i += swarm.HashSize
+
+	c.ChunkType = swarm.ChunkType(buf[i])
+
+	return nil
+}

pkg/storer/internal/stampindex/export_test.go

@@ -14,16 +14,15 @@ var (
 )
 
 // NewItemWithValues creates a new Item with given values and fixed keys.
-func NewItemWithValues(batchTimestamp []byte, chunkAddress swarm.Address, chunkIsImmutable bool) *Item {
+func NewItemWithValues(batchTimestamp []byte, chunkAddress swarm.Address) *Item {
 	return &Item{
 		namespace:  []byte("test_namespace"),
 		BatchID:    []byte{swarm.HashSize - 1: 9},
 		StampIndex: []byte{swarm.StampIndexSize - 1: 9},
 		StampHash:  swarm.EmptyAddress.Bytes(),
 
-		StampTimestamp:   batchTimestamp,
-		ChunkAddress:     chunkAddress,
-		ChunkIsImmutable: chunkIsImmutable,
+		StampTimestamp: batchTimestamp,
+		ChunkAddress:   chunkAddress,
 	}
 }
 

pkg/storer/internal/stampindex/oldstampindex.go

@@ -0,0 +1,115 @@
+// Copyright 2024 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stampindex
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/ethersphere/bee/v2/pkg/storage"
+	"github.com/ethersphere/bee/v2/pkg/storage/storageutil"
+	"github.com/ethersphere/bee/v2/pkg/storer/internal"
+	"github.com/ethersphere/bee/v2/pkg/swarm"
+)
+
+// ItemV1 is an store.Item that represents data relevant to stamp.
+type ItemV1 struct {
+	// Keys.
+	namespace  []byte // The namespace of other related item.
+	BatchID    []byte
+	StampIndex []byte
+
+	// Values.
+	StampTimestamp   []byte
+	ChunkAddress     swarm.Address
+	ChunkIsImmutable bool
+}
+
+// ID implements the storage.Item interface.
+func (i ItemV1) ID() string {
+	return fmt.Sprintf("%s/%s/%s", string(i.namespace), string(i.BatchID), string(i.StampIndex))
+}
+
+// Namespace implements the storage.Item interface.
+func (i ItemV1) Namespace() string {
+	return "stampIndex"
+}
+
+// Marshal implements the storage.Item interface.
+func (i ItemV1) Marshal() ([]byte, error) {
+	switch {
+	case len(i.namespace) == 0:
+		return nil, errStampItemMarshalNamespaceInvalid
+	case len(i.BatchID) != swarm.HashSize:
+		return nil, errStampItemMarshalBatchIDInvalid
+	case len(i.StampIndex) != swarm.StampIndexSize:
+		return nil, errStampItemMarshalBatchIndexInvalid
+	}
+
+	buf := make([]byte, 8+len(i.namespace)+swarm.HashSize+swarm.StampIndexSize+swarm.StampTimestampSize+swarm.HashSize)
+
+	l := 0
+	binary.LittleEndian.PutUint64(buf[l:l+8], uint64(len(i.namespace)))
+	l += 8
+	copy(buf[l:l+len(i.namespace)], i.namespace)
+	l += len(i.namespace)
+	copy(buf[l:l+swarm.HashSize], i.BatchID)
+	l += swarm.HashSize
+	copy(buf[l:l+swarm.StampIndexSize], i.StampIndex)
+	l += swarm.StampIndexSize
+	copy(buf[l:l+swarm.StampTimestampSize], i.StampTimestamp)
+	l += swarm.StampTimestampSize
+	copy(buf[l:l+swarm.HashSize], internal.AddressBytesOrZero(i.ChunkAddress))
+	return buf, nil
+}
+
+// Unmarshal implements the storage.Item interface.
+func (i *ItemV1) Unmarshal(bytes []byte) error {
+	if len(bytes) < 8 {
+		return errStampItemUnmarshalInvalidSize
+	}
+	nsLen := int(binary.LittleEndian.Uint64(bytes))
+	if len(bytes) != 8+nsLen+swarm.HashSize+swarm.StampIndexSize+swarm.StampTimestampSize+swarm.HashSize {
+		return errStampItemUnmarshalInvalidSize
+	}
+
+	ni := new(ItemV1)
+	l := 8
+	ni.namespace = append(make([]byte, 0, nsLen), bytes[l:l+nsLen]...)
+	l += nsLen
+	ni.BatchID = append(make([]byte, 0, swarm.HashSize), bytes[l:l+swarm.HashSize]...)
+	l += swarm.HashSize
+	ni.StampIndex = append(make([]byte, 0, swarm.StampIndexSize), bytes[l:l+swarm.StampIndexSize]...)
+	l += swarm.StampIndexSize
+	ni.StampTimestamp = append(make([]byte, 0, swarm.StampTimestampSize), bytes[l:l+swarm.StampTimestampSize]...)
+	l += swarm.StampTimestampSize
+	ni.ChunkAddress = internal.AddressOrZero(bytes[l : l+swarm.HashSize])
+	*i = *ni
+	return nil
+}
+
+// Clone  implements the storage.Item interface.
+func (i *ItemV1) Clone() storage.Item {
+	if i == nil {
+		return nil
+	}
+	return &ItemV1{
+		namespace:        append([]byte(nil), i.namespace...),
+		BatchID:          append([]byte(nil), i.BatchID...),
+		StampIndex:       append([]byte(nil), i.StampIndex...),
+		StampTimestamp:   append([]byte(nil), i.StampTimestamp...),
+		ChunkAddress:     i.ChunkAddress.Clone(),
+		ChunkIsImmutable: i.ChunkIsImmutable,
+	}
+}
+
+// String implements the fmt.Stringer interface.
+func (i ItemV1) String() string {
+	return storageutil.JoinFields(i.Namespace(), i.ID())
+}
+
+func (i *ItemV1) SetNamespace(ns []byte) {
+	i.namespace = ns
+}