feat(storageincentives): storage incentives phase 4 (#4345)

Co-authored-by: Viktor Levente Tóth <[email protected]>
Co-authored-by: nugaon <[email protected]>
Co-authored-by: istae <[email protected]>

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/ethereum/go-ethereum v1.12.2
 	github.com/ethersphere/go-price-oracle-abi v0.1.0
-	github.com/ethersphere/go-storage-incentives-abi v0.5.0
+	github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3
 	github.com/ethersphere/go-sw3-abi v0.4.0
 	github.com/ethersphere/langos v1.0.0
 	github.com/go-playground/validator/v10 v10.11.1

go.sum

@@ -223,8 +223,8 @@ github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6
 github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI=
 github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8=
 github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk=
-github.com/ethersphere/go-storage-incentives-abi v0.5.0 h1:dd01OZmPraCjOIiSX5FsCfFFwUR2b9PuTO/LDcYxS+s=
-github.com/ethersphere/go-storage-incentives-abi v0.5.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc=
+github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3 h1:tXux2FnhuU6DbrY+Z4nVQMGp63JkJPq7pKb5Xi2Sjxo=
+github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc=
 github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g=
 github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU=
 github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=

pkg/api/api.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -286,6 +287,10 @@ func New(
 			buf, err := base64.URLEncoding.DecodeString(v)
 			return string(buf), err
 		},
+		"decHex": func(v string) (string, error) {
+			buf, err := hex.DecodeString(v)
+			return string(buf), err
+		},
 	}
 	s.validate = validator.New()
 	s.validate.RegisterTagNameFunc(func(fld reflect.StructField) string {

pkg/api/api_test.go

@@ -57,6 +57,7 @@ import (
 	"github.com/ethersphere/bee/pkg/storage/inmemstore"
 	testingc "github.com/ethersphere/bee/pkg/storage/testing"
 	"github.com/ethersphere/bee/pkg/storageincentives"
+	"github.com/ethersphere/bee/pkg/storageincentives/redistribution"
 	"github.com/ethersphere/bee/pkg/storageincentives/staking"
 	mock2 "github.com/ethersphere/bee/pkg/storageincentives/staking/mock"
 	mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
@@ -775,14 +776,14 @@ func (m *mockContract) IsWinner(context.Context) (bool, error) {
 	return false, nil
 }
 
-func (m *mockContract) Claim(context.Context) (common.Hash, error) {
+func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) {
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 	m.callsList = append(m.callsList, claimCall)
 	return common.Hash{}, nil
 }
 
-func (m *mockContract) Commit(context.Context, []byte, *big.Int) (common.Hash, error) {
+func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) {
 	m.mtx.Lock()
 	defer m.mtx.Unlock()
 	m.callsList = append(m.callsList, commitCall)

pkg/api/rchash.go

@@ -6,40 +6,135 @@ package api
 import (
 	"encoding/hex"
 	"net/http"
+	"strconv"
+	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
-	"github.com/ethersphere/bee/pkg/storageincentives"
+	"github.com/ethersphere/bee/pkg/storageincentives/redistribution"
+	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/gorilla/mux"
 )
 
-type RCHashResponse storageincentives.SampleWithProofs
+type RCHashResponse struct {
+	Hash     swarm.Address        `json:"hash"`
+	Proofs   ChunkInclusionProofs `json:"proofs"`
+	Duration time.Duration        `json:"duration"`
+}
+
+type ChunkInclusionProofs struct {
+	A ChunkInclusionProof `json:"proof1"`
+	B ChunkInclusionProof `json:"proof2"`
+	C ChunkInclusionProof `json:"proofLast"`
+}
+
+// ChunkInclusionProof structure must exactly match
+// corresponding structure (of the same name) in Redistribution.sol smart contract.
+// github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol
+// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master)
+type ChunkInclusionProof struct {
+	ProofSegments  []string     `json:"proofSegments"`
+	ProveSegment   string       `json:"proveSegment"`
+	ProofSegments2 []string     `json:"proofSegments2"`
+	ProveSegment2  string       `json:"proveSegment2"`
+	ChunkSpan      uint64       `json:"chunkSpan"`
+	ProofSegments3 []string     `json:"proofSegments3"`
+	PostageProof   PostageProof `json:"postageProof"`
+	SocProof       []SOCProof   `json:"socProof"`
+}
+
+// SOCProof structure must exactly match
+// corresponding structure (of the same name) in Redistribution.sol smart contract.
+type PostageProof struct {
+	Signature string `json:"signature"`
+	PostageId string `json:"postageId"`
+	Index     string `json:"index"`
+	TimeStamp string `json:"timeStamp"`
+}
+
+// SOCProof structure must exactly match
+// corresponding structure (of the same name) in Redistribution.sol smart contract.
+type SOCProof struct {
+	Signer     string `json:"signer"`
+	Signature  string `json:"signature"`
+	Identifier string `json:"identifier"`
+	ChunkAddr  string `json:"chunkAddr"`
+}
+
+func renderChunkInclusionProofs(proofs redistribution.ChunkInclusionProofs) ChunkInclusionProofs {
+	return ChunkInclusionProofs{
+		A: renderChunkInclusionProof(proofs.A),
+		B: renderChunkInclusionProof(proofs.B),
+		C: renderChunkInclusionProof(proofs.C),
+	}
+}
+
+func renderChunkInclusionProof(proof redistribution.ChunkInclusionProof) ChunkInclusionProof {
+	var socProof []SOCProof
+	if len(proof.SocProof) == 1 {
+		socProof = []SOCProof{{
+			Signer:     hex.EncodeToString(proof.SocProof[0].Signer.Bytes()),
+			Signature:  hex.EncodeToString(proof.SocProof[0].Signature[:]),
+			Identifier: hex.EncodeToString(proof.SocProof[0].Identifier.Bytes()),
+			ChunkAddr:  hex.EncodeToString(proof.SocProof[0].ChunkAddr.Bytes()),
+		}}
+	}
+
+	return ChunkInclusionProof{
+		ProveSegment:   hex.EncodeToString(proof.ProveSegment.Bytes()),
+		ProofSegments:  renderCommonHash(proof.ProofSegments),
+		ProveSegment2:  hex.EncodeToString(proof.ProveSegment2.Bytes()),
+		ProofSegments2: renderCommonHash(proof.ProofSegments2),
+		ProofSegments3: renderCommonHash(proof.ProofSegments3),
+		ChunkSpan:      proof.ChunkSpan,
+		PostageProof: PostageProof{
+			Signature: hex.EncodeToString(proof.PostageProof.Signature[:]),
+			PostageId: hex.EncodeToString(proof.PostageProof.PostageId[:]),
+			Index:     strconv.FormatUint(proof.PostageProof.Index, 16),
+			TimeStamp: strconv.FormatUint(proof.PostageProof.TimeStamp, 16),
+		},
+		SocProof: socProof,
+	}
+}
+
+func renderCommonHash(proofSegments []common.Hash) []string {
+	output := make([]string, len(proofSegments))
+	for i, s := range proofSegments {
+		output[i] = hex.EncodeToString(s.Bytes())
+	}
+	return output
+}
 
 // This API is kept for testing the sampler. As a result, no documentation or tests are added here.
 func (s *Service) rchash(w http.ResponseWriter, r *http.Request) {
 	logger := s.logger.WithName("get_rchash").Build()
 
 	paths := struct {
-		Depth   *uint8 `map:"depth" validate:"required"`
-		Anchor1 string `map:"anchor1" validate:"required"`
+		Depth   uint8  `map:"depth"`
+		Anchor1 string `map:"anchor1,decHex" validate:"required"`
+		Anchor2 string `map:"anchor2,decHex" validate:"required"`
 	}{}
 	if response := s.mapStructure(mux.Vars(r), &paths); response != nil {
 		response("invalid path params", logger, w)
 		return
 	}
 
-	anchor1, err := hex.DecodeString(paths.Anchor1)
-	if err != nil {
-		logger.Error(err, "invalid hex params")
-		jsonhttp.InternalServerError(w, "invalid hex params")
-		return
-	}
+	anchor1 := []byte(paths.Anchor1)
 
-	resp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, *paths.Depth)
+	anchor2 := []byte(paths.Anchor2)
+
+	swp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth)
 	if err != nil {
 		logger.Error(err, "failed making sample with proofs")
 		jsonhttp.InternalServerError(w, "failed making sample with proofs")
 		return
 	}
 
-	jsonhttp.OK(w, RCHashResponse(resp))
+	resp := RCHashResponse{
+		Hash:     swp.Hash,
+		Duration: swp.Duration,
+		Proofs:   renderChunkInclusionProofs(swp.Proofs),
+	}
+
+	jsonhttp.OK(w, resp)
 }

pkg/api/router.go

@@ -339,12 +339,6 @@ func (s *Service) mountAPI() {
 		web.FinalHandlerFunc(s.healthHandler),
 	))
 
-	handle("/rchash/{depth}/{anchor1}", web.ChainHandlers(
-		web.FinalHandler(jsonhttp.MethodHandler{
-			"GET": http.HandlerFunc(s.rchash),
-		}),
-	))
-
 	if s.Restricted {
 		handle("/auth", jsonhttp.MethodHandler{
 			"POST": web.ChainHandlers(
@@ -601,4 +595,10 @@ func (s *Service) mountBusinessDebug(restricted bool) {
 			web.FinalHandlerFunc(s.statusGetPeersHandler),
 		),
 	})
+
+	handle("/rchash/{depth}/{anchor1}/{anchor2}", web.ChainHandlers(
+		web.FinalHandler(jsonhttp.MethodHandler{
+			"GET": http.HandlerFunc(s.rchash),
+		}),
+	))
 }

pkg/bmt/bmt.go

@@ -40,6 +40,19 @@ type Hasher struct {
 	span   []byte      // The span of the data subsumed under the chunk
 }
 
+// NewHasher gives back an instance of a Hasher struct
+func NewHasher(hasherFact func() hash.Hash) *Hasher {
+	conf := NewConf(hasherFact, swarm.BmtBranches, 32)
+
+	return &Hasher{
+		Conf:   conf,
+		result: make(chan []byte),
+		errc:   make(chan error, 1),
+		span:   make([]byte, SpanSize),
+		bmt:    newTree(conf.segmentSize, conf.maxSize, conf.depth, conf.hasher),
+	}
+}
+
 // Capacity returns the maximum amount of bytes that will be processed by this hasher implementation.
 // since BMT assumes a balanced binary tree, capacity it is always a power of 2
 func (h *Hasher) Capacity() int {

pkg/bmt/proof.go

@@ -17,6 +17,17 @@ type Proof struct {
 	Index         int
 }
 
+// Hash overrides base hash function of Hasher to fill buffer with zeros until chunk length
+func (p Prover) Hash(b []byte) ([]byte, error) {
+	for i := p.size; i < p.maxSize; i += len(zerosection) {
+		_, err := p.Hasher.Write(zerosection)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return p.Hasher.Hash(b)
+}
+
 // Proof returns the inclusion proof of the i-th data segment
 func (p Prover) Proof(i int) Proof {
 	index := i
@@ -36,34 +47,46 @@ func (p Prover) Proof(i int) Proof {
 	secsize := 2 * p.segmentSize
 	offset := i * secsize
 	section := p.bmt.buffer[offset : offset+secsize]
-	return Proof{section, sisters, p.span, index}
+	segment, firstSegmentSister := section[:p.segmentSize], section[p.segmentSize:]
+	if index%2 != 0 {
+		segment, firstSegmentSister = firstSegmentSister, segment
+	}
+	sisters = append([][]byte{firstSegmentSister}, sisters...)
+	return Proof{segment, sisters, p.span, index}
 }
 
 // Verify returns the bmt hash obtained from the proof which can then be checked against
 // the BMT hash of the chunk
 func (p Prover) Verify(i int, proof Proof) (root []byte, err error) {
+	var section []byte
+	if i%2 == 0 {
+		section = append(append(section, proof.ProveSegment...), proof.ProofSegments[0]...)
+	} else {
+		section = append(append(section, proof.ProofSegments[0]...), proof.ProveSegment...)
+	}
 	i = i / 2
 	n := p.bmt.leaves[i]
+	hasher := p.hasher()
 	isLeft := n.isLeft
-	root, err = doHash(n.hasher, proof.ProveSegment)
+	root, err = doHash(hasher, section)
 	if err != nil {
 		return nil, err
 	}
 	n = n.parent
 
-	for _, sister := range proof.ProofSegments {
+	for _, sister := range proof.ProofSegments[1:] {
 		if isLeft {
-			root, err = doHash(n.hasher, root, sister)
+			root, err = doHash(hasher, root, sister)
 		} else {
-			root, err = doHash(n.hasher, sister, root)
+			root, err = doHash(hasher, sister, root)
 		}
 		if err != nil {
 			return nil, err
 		}
 		isLeft = n.isLeft
 		n = n.parent
 	}
-	return sha3hash(proof.Span, root)
+	return doHash(hasher, proof.Span, root)
 }
 
 func (n *node) getSister(isLeft bool) []byte {