test: refactor to allow benchmark Vote (WIP, MVP)

.github/workflows/main.yml

@@ -94,6 +94,18 @@ jobs:
         if: github.event_name == 'push' # this is limited to selected branches at the beginning of this file
         run: go test -coverprofile=unit.covdata.txt -vet=off -timeout=15m -race ./... # note that -race can easily make the crypto stuff 10x slower
 
+  job_go_bench:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Go environment
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - name: Run Go bench
+        run: go test -run='^$' -bench=. ./...
+
   job_compose_test:
     runs-on: [self-hosted, ci2-1]
     steps:

test/api_test.go

@@ -10,6 +10,7 @@ import (
 	qt "github.com/frankban/quicktest"
 	"github.com/google/uuid"
 	"go.vocdoni.io/dvote/api"
+	"go.vocdoni.io/dvote/apiclient"
 	"go.vocdoni.io/dvote/crypto/ethereum"
 	"go.vocdoni.io/dvote/data/ipfs"
 	"go.vocdoni.io/dvote/test/testcommon"
@@ -22,9 +23,43 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+type testElection struct {
+	server     testcommon.APIserver
+	c          *testutil.TestHTTPclient
+	election   api.ElectionCreate
+	censusData *api.Census
+
+	voters []voter
+}
+
+type voter struct {
+	key   *ethereum.SignKeys
+	proof *apiclient.CensusProof
+	vote  *api.Vote
+}
+
 func TestAPIcensusAndVote(t *testing.T) {
-	server := testcommon.APIserver{}
-	server.Start(t,
+	te := NewTestElection(t, t.TempDir())
+	te.GenerateVoters(t, 10)
+	te.CreateCensusAndElection(t)
+	// Block 2
+	te.server.VochainAPP.AdvanceTestBlock()
+
+	te.VoteAll(t)
+
+	// Block 3
+	te.server.VochainAPP.AdvanceTestBlock()
+	//waitUntilHeight(t, te.c, 3)
+
+	te.VerifyVotes(t)
+}
+
+func NewTestElection(t testing.TB, datadir string) *testElection {
+	te := &testElection{}
+
+	// Server
+	te.server = testcommon.APIserver{}
+	te.server.Start(t, datadir,
 		api.ChainHandler,
 		api.CensusHandler,
 		api.VoteHandler,
@@ -33,50 +68,77 @@ func TestAPIcensusAndVote(t *testing.T) {
 		api.WalletHandler,
 	)
 	// Block 1
-	server.VochainAPP.AdvanceTestBlock()
+	te.server.VochainAPP.AdvanceTestBlock()
 
+	// Client
 	token1 := uuid.New()
-	c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1)
+	te.c = testutil.NewTestHTTPclient(t, te.server.ListenAddr, &token1)
+	return te
+}
 
-	// create a new census
-	resp, code := c.Request("POST", nil, "censuses", "weighted")
-	qt.Assert(t, code, qt.Equals, 200)
-	censusData := &api.Census{}
-	qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil)
-	id1 := censusData.CensusID.String()
-
-	// add a bunch of keys and values (weights)
-	rnd := testutil.NewRandom(1)
-	cparts := api.CensusParticipants{}
-	for i := 1; i < 10; i++ {
-		cparts.Participants = append(cparts.Participants, api.CensusParticipant{
-			Key:    rnd.RandomBytes(20),
-			Weight: (*types.BigInt)(big.NewInt(int64(1))),
-		})
+func (te *testElection) GenerateVoters(t testing.TB, nvotes int) {
+	// Voters
+	te.voters = nil
+	for i := 0; i < nvotes; i++ {
+		k := ethereum.NewSignKeys()
+		qt.Assert(t, k.Generate(), qt.IsNil)
+		te.voters = append(te.voters, voter{key: k})
 	}
-	_, code = c.Request("POST", &cparts, "censuses", id1, "participants")
-	qt.Assert(t, code, qt.Equals, 200)
+}
 
-	// add the key we'll use for cast votes
-	voterKey := ethereum.SignKeys{}
-	qt.Assert(t, voterKey.Generate(), qt.IsNil)
+func (te *testElection) AddCensusParticipants(t testing.TB, id string) types.HexBytes {
+	// add a bunch of keys and values (weights) in batches of api.MaxCensusAddBatchSize
+	for i := 0; i < len(te.voters); i += api.MaxCensusAddBatchSize {
+		// Get the next chunk of voters
+		end := i + api.MaxCensusAddBatchSize
+		if end > len(te.voters) {
+			end = len(te.voters)
+		}
 
-	_, code = c.Request("POST", &api.CensusParticipants{Participants: []api.CensusParticipant{{
-		Key:    voterKey.Address().Bytes(),
-		Weight: (*types.BigInt)(big.NewInt(1)),
-	}}}, "censuses", id1, "participants")
-	qt.Assert(t, code, qt.Equals, 200)
+		// Add the voters in the chunk to the cparts struct
+		cparts := api.CensusParticipants{}
+		for _, voter := range te.voters[i:end] {
+			cparts.Participants = append(cparts.Participants, api.CensusParticipant{
+				Key:    voter.key.Address().Bytes(),
+				Weight: (*types.BigInt)(big.NewInt(1)),
+			})
+		}
+
+		// POST this chunk of voters
+		resp, code := te.c.Request("POST", &cparts, "censuses", id, "participants")
+		qt.Assert(t, code, qt.Equals, 200)
+		qt.Assert(t, resp, qt.IsNotNil)
+	}
 
-	resp, code = c.Request("POST", nil, "censuses", id1, "publish")
+	resp, code := te.c.Request("POST", nil, "censuses", id, "publish")
 	qt.Assert(t, code, qt.Equals, 200)
-	qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil)
-	qt.Assert(t, censusData.CensusID, qt.IsNotNil)
-	root := censusData.CensusID
+	qt.Assert(t, json.Unmarshal(resp, te.censusData), qt.IsNil)
+	qt.Assert(t, te.censusData.CensusID, qt.IsNotNil)
+	root := te.censusData.CensusID
+	return root
+}
 
-	resp, code = c.Request("GET", nil, "censuses", root.String(), "proof", fmt.Sprintf("%x", voterKey.Address().Bytes()))
+func (te *testElection) CreateCensusAndElection(t testing.TB) {
+	// create a new census
+	resp, code := te.c.Request("POST", nil, "censuses", "weighted")
 	qt.Assert(t, code, qt.Equals, 200)
-	qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil)
-	qt.Assert(t, censusData.Weight.String(), qt.Equals, "1")
+	te.censusData = &api.Census{}
+	qt.Assert(t, json.Unmarshal(resp, te.censusData), qt.IsNil)
+	id1 := te.censusData.CensusID.String()
+
+	root := te.AddCensusParticipants(t, id1)
+	for i, voter := range te.voters {
+		censusData := &api.Census{}
+		resp, code = te.c.Request("GET", nil, "censuses", root.String(),
+			"proof", fmt.Sprintf("%x", voter.key.Address().Bytes()))
+		qt.Assert(t, code, qt.Equals, 200)
+		qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil)
+		qt.Assert(t, censusData.Weight.String(), qt.Equals, "1")
+		te.voters[i].proof = &apiclient.CensusProof{
+			Proof:     censusData.Proof,
+			LeafValue: censusData.Value,
+		}
+	}
 
 	metadataBytes, err := json.Marshal(
 		&api.ElectionMetadata{
@@ -103,32 +165,38 @@ func TestAPIcensusAndVote(t *testing.T) {
 					VoteOptions:   &models.ProcessVoteOptions{MaxCount: 1, MaxValue: 1},
 					EnvelopeType:  &models.EnvelopeType{},
 					Metadata:      &metadataURI,
-					MaxCensusSize: 1000,
+					MaxCensusSize: uint64(len(te.voters)),
 				},
 			},
 		},
 	}
 	txb, err := proto.Marshal(&tx)
 	qt.Assert(t, err, qt.IsNil)
-	signedTxb, err := server.Account.SignVocdoniTx(txb, server.VochainAPP.ChainID())
+	signedTxb, err := te.server.Account.SignVocdoniTx(txb, te.server.VochainAPP.ChainID())
 	qt.Assert(t, err, qt.IsNil)
 	stx := models.SignedTx{Tx: txb, Signature: signedTxb}
 	stxb, err := proto.Marshal(&stx)
 	qt.Assert(t, err, qt.IsNil)
 
-	election := api.ElectionCreate{
+	te.election = api.ElectionCreate{
 		TxPayload: stxb,
 		Metadata:  metadataBytes,
 	}
-	resp, code = c.Request("POST", election, "elections")
+	te.server.AccountInit(t)
+	resp, code = te.c.Request("POST", te.election, "elections")
 	qt.Assert(t, code, qt.Equals, 200)
-	err = json.Unmarshal(resp, &election)
+	err = json.Unmarshal(resp, &te.election)
 	qt.Assert(t, err, qt.IsNil)
+}
 
-	// Block 2
-	server.VochainAPP.AdvanceTestBlock()
-	waitUntilHeight(t, c, 2)
+func (te *testElection) VoteAll(t testing.TB) {
+	for i, voter := range te.voters {
+		te.voters[i].vote = te.Vote(t, voter)
+	}
+}
 
+// Vote sends a vote
+func (te *testElection) Vote(t testing.TB, voter voter) *api.Vote {
 	// Send a vote
 	votePackage := &state.VotePackage{
 		Votes: []int{1},
@@ -138,56 +206,58 @@ func TestAPIcensusAndVote(t *testing.T) {
 
 	vote := &models.VoteEnvelope{
 		Nonce:       util.RandomBytes(16),
-		ProcessId:   election.ElectionID,
+		ProcessId:   te.election.ElectionID,
 		VotePackage: votePackageBytes,
 	}
 	vote.Proof = &models.Proof{
 		Payload: &models.Proof_Arbo{
 			Arbo: &models.ProofArbo{
 				Type:       models.ProofArbo_BLAKE2B,
-				Siblings:   censusData.Proof,
-				LeafWeight: censusData.Value,
+				Siblings:   voter.proof.Proof,
+				LeafWeight: voter.proof.LeafValue,
 			},
 		},
 	}
+	stx := models.SignedTx{}
 	stx.Tx, err = proto.Marshal(&models.Tx{Payload: &models.Tx_Vote{Vote: vote}})
 	qt.Assert(t, err, qt.IsNil)
-	stx.Signature, err = voterKey.SignVocdoniTx(stx.Tx, server.VochainAPP.ChainID())
+	stx.Signature, err = voter.key.SignVocdoniTx(stx.Tx, te.server.VochainAPP.ChainID())
 	qt.Assert(t, err, qt.IsNil)
-	stxb, err = proto.Marshal(&stx)
+	stxb, err := proto.Marshal(&stx)
 	qt.Assert(t, err, qt.IsNil)
 
 	v := &api.Vote{TxPayload: stxb}
-	resp, code = c.Request("POST", v, "votes")
+	resp, code := te.c.Request("POST", v, "votes")
+	if code == 500 {
+		t.Logf("%s", resp)
+	}
 	qt.Assert(t, code, qt.Equals, 200)
 	err = json.Unmarshal(resp, &v)
 	qt.Assert(t, err, qt.IsNil)
+	return v
+}
 
-	// Block 3
-	server.VochainAPP.AdvanceTestBlock()
-	waitUntilHeight(t, c, 3)
-
-	// Verify the vote
-	_, code = c.Request("GET", nil, "votes", "verify", election.ElectionID.String(), v.VoteID.String())
-	qt.Assert(t, code, qt.Equals, 200)
-
-	// Get the vote and check the data
-	resp, code = c.Request("GET", nil, "votes", v.VoteID.String())
-	qt.Assert(t, code, qt.Equals, 200)
-	v2 := &api.Vote{}
-	err = json.Unmarshal(resp, v2)
-	qt.Assert(t, err, qt.IsNil)
-	qt.Assert(t, v2.VoteID.String(), qt.Equals, v.VoteID.String())
-	qt.Assert(t, v2.BlockHeight, qt.Equals, uint32(2))
-	qt.Assert(t, *v2.TransactionIndex, qt.Equals, int32(0))
+func (te *testElection) VerifyVotes(t testing.TB) {
+	for _, voter := range te.voters {
+		// Verify the vote
+		_, code := te.c.Request("GET", nil, "votes", "verify",
+			te.election.ElectionID.String(), voter.vote.VoteID.String())
+		qt.Assert(t, code, qt.Equals, 200)
 
-	// TODO (painan): check why the voterID is not present on the reply
-	//qt.Assert(t, v2.VoterID.String(), qt.Equals, voterKey.AddressString())
+		// Get the vote and check the data
+		resp, code := te.c.Request("GET", nil, "votes", voter.vote.VoteID.String())
+		qt.Assert(t, code, qt.Equals, 200)
+		v2 := &api.Vote{}
+		err := json.Unmarshal(resp, v2)
+		qt.Assert(t, err, qt.IsNil)
+		qt.Assert(t, v2.VoteID.String(), qt.Equals, voter.vote.VoteID.String())
+		qt.Assert(t, v2.VoterID.String(), qt.Equals, fmt.Sprintf("%x", voter.key.Address().Bytes()))
+	}
 }
 
 func TestAPIaccount(t *testing.T) {
 	server := testcommon.APIserver{}
-	server.Start(t,
+	server.Start(t, t.TempDir(),
 		api.ChainHandler,
 		api.CensusHandler,
 		api.VoteHandler,

test/apierror_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestAPIerror(t *testing.T) {
 	server := testcommon.APIserver{}
-	server.Start(t,
+	server.Start(t, t.TempDir(),
 		api.ChainHandler,
 		api.CensusHandler,
 		api.VoteHandler,