From d132d5708c728b825d7cf3ce1a1b5a9122659a63 Mon Sep 17 00:00:00 2001 From: "Lucas\\ Menendez" Date: Thu, 10 Aug 2023 14:10:16 +0200 Subject: [PATCH 01/15] support for create elections with tempSIKs flag, temp SIKs will be purged by the IST after results calculation --- api/api_types.go | 1 + apiclient/election.go | 1 + apiclient/sik.go | 13 ++++ cmd/end2endtest/censusize.go | 2 +- cmd/end2endtest/dynamicensus.go | 6 +- cmd/end2endtest/encrypted.go | 2 +- cmd/end2endtest/helpers.go | 18 ++--- cmd/end2endtest/lifecycle.go | 2 +- cmd/end2endtest/overwrite.go | 2 +- cmd/end2endtest/plaintext.go | 2 +- cmd/end2endtest/tempsiks.go | 120 +++++++++++++++++++++++++++++ cmd/end2endtest/zkweighted.go | 2 +- vochain/ist/ist.go | 13 +++- vochain/state/sik.go | 23 +++--- vochain/transaction/transaction.go | 2 + vochain/transaction_zk_test.go | 2 +- 16 files changed, 180 insertions(+), 31 deletions(-) create mode 100644 cmd/end2endtest/tempsiks.go diff --git a/api/api_types.go b/api/api_types.go index aed38e9e1..28f248a9e 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -95,6 +95,7 @@ type ElectionDescription struct { ElectionType ElectionType `json:"electionType"` Questions []Question `json:"questions"` Census CensusTypeDescription `json:"census"` + TempSIKs bool `json:"tempSIKs"` } type ElectionFilter struct { diff --git a/apiclient/election.go b/apiclient/election.go index 8ac03edd0..4b96efc26 100644 --- a/apiclient/election.go +++ b/apiclient/election.go @@ -214,6 +214,7 @@ func (c *HTTPclient) NewElection(description *api.ElectionDescription) (types.He CensusOrigin: censusOrigin, Metadata: &metadataURI, MaxCensusSize: description.Census.Size, + TempSIKs: &description.TempSIKs, } log.Debugf("election transaction: %+v", log.FormatProto(process)) diff --git a/apiclient/sik.go b/apiclient/sik.go index 70df62268..c41d7dfcb 100644 --- a/apiclient/sik.go +++ b/apiclient/sik.go @@ -10,6 +10,19 @@ import ( type SIKRoots []string +// ValidSIK checks if the current client account has a valid SIK registered in +// the vochain +func (c *HTTPclient) ValidSIK() (bool, error) { + resp, code, err := c.Request(HTTPGET, nil, "siks", c.account.AddressString()) + if err != nil { + return false, err + } + if code != apirest.HTTPstatusOK { + return false, fmt.Errorf("%s: %d (%s)", errCodeNot200, code, resp) + } + return true, nil +} + // ValidSIKRoots returns the currently valid roots of SIK merkle tree from the // API. func (c *HTTPclient) ValidSIKRoots() (SIKRoots, error) { diff --git a/cmd/end2endtest/censusize.go b/cmd/end2endtest/censusize.go index 48fc60e28..dad8a784f 100644 --- a/cmd/end2endtest/censusize.go +++ b/cmd/end2endtest/censusize.go @@ -41,7 +41,7 @@ func (t *E2EMaxCensusSizeElection) Setup(api *apiclient.HTTPclient, c *config) e Size: uint64(t.config.nvotes - 1), } - if err := t.setupElection(ed); err != nil { + if err := t.setupElection(ed, false); err != nil { return err } diff --git a/cmd/end2endtest/dynamicensus.go b/cmd/end2endtest/dynamicensus.go index f92675888..15b8262f4 100644 --- a/cmd/end2endtest/dynamicensus.go +++ b/cmd/end2endtest/dynamicensus.go @@ -48,7 +48,7 @@ func (t *E2EDynamicensusElection) Setup(api *apiclient.HTTPclient, c *config) er // create a census with 2 voterAccounts less than the nvotes passed, that will allow to create another // census with the missing nvotes - censusRoot, censusURI, err := t.elections[i].setupCensus(vapi.CensusTypeWeighted, t.elections[i].config.nvotes-2) + censusRoot, censusURI, err := t.elections[i].setupCensus(vapi.CensusTypeWeighted, t.elections[i].config.nvotes-2, false) if err != nil { return err } @@ -62,7 +62,7 @@ func (t *E2EDynamicensusElection) Setup(api *apiclient.HTTPclient, c *config) er } // set up the election with the custom census created - if err := t.elections[i].setupElection(ed.d); err != nil { + if err := t.elections[i].setupElection(ed.d, false); err != nil { return err } log.Debugf("election detail: %+v", *t.elections[i].election) @@ -84,7 +84,7 @@ func (t *E2EDynamicensusElection) Run() error { api := election.api nvotes := election.config.nvotes - censusRoot2, censusURI2, err := election.setupCensus(vapi.CensusTypeWeighted, 2) + censusRoot2, censusURI2, err := election.setupCensus(vapi.CensusTypeWeighted, 2, false) if err != nil { return nil, apiclient.VoteData{}, err } diff --git a/cmd/end2endtest/encrypted.go b/cmd/end2endtest/encrypted.go index 526a7dc01..e6dd77997 100644 --- a/cmd/end2endtest/encrypted.go +++ b/cmd/end2endtest/encrypted.go @@ -36,7 +36,7 @@ func (t *E2EEncryptedElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed); err != nil { + if err := t.setupElection(ed, false); err != nil { return err } diff --git a/cmd/end2endtest/helpers.go b/cmd/end2endtest/helpers.go index 62f2bce81..a00fb879e 100644 --- a/cmd/end2endtest/helpers.go +++ b/cmd/end2endtest/helpers.go @@ -167,7 +167,7 @@ func (t *e2eElection) waitUntilElectionStarts(electionID types.HexBytes) (*vapi. return election, nil } -func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool, csp *ethereum.SignKeys) (map[string]*apiclient.CensusProof, map[string]*apiclient.CensusProof) { +func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool, csp *ethereum.SignKeys, generateSIKProofs bool) (map[string]*apiclient.CensusProof, map[string]*apiclient.CensusProof) { type voterProof struct { proof *apiclient.CensusProof sikproof *apiclient.CensusProof @@ -205,7 +205,7 @@ func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool if err != nil { log.Warn(err) } - if isAnonymousVoting { + if generateSIKProofs { voterProof.sikproof, err = voterApi.GenSIKProof() } } @@ -249,7 +249,7 @@ func (t *e2eElection) setupAccount() error { } // setupCensus create a new census that will have nAcct voterAccounts and participants -func (t *e2eElection) setupCensus(censusType string, nAcct int) (types.HexBytes, string, error) { +func (t *e2eElection) setupCensus(censusType string, nAcct int, createAccounts bool) (types.HexBytes, string, error) { // Create a new census censusID, err := t.api.NewCensus(censusType) if err != nil { @@ -261,7 +261,7 @@ func (t *e2eElection) setupCensus(censusType string, nAcct int) (types.HexBytes, t.voterAccounts = append(t.voterAccounts, ethereum.NewSignKeysBatch(nAcct)...) // Register the accounts in the vochain if is required - if censusType == vapi.CensusTypeZKWeighted { + if createAccounts { for _, acc := range t.voterAccounts { pKey := acc.PrivateKey() if _, _, err := t.createAccount(pKey.String()); err != nil && @@ -300,7 +300,7 @@ func (t *e2eElection) setupCensus(censusType string, nAcct int) (types.HexBytes, return censusRoot, censusURI, nil } -func (t *e2eElection) setupElection(ed *vapi.ElectionDescription) error { +func (t *e2eElection) setupElection(ed *vapi.ElectionDescription, createAccounts bool) error { if err := t.setupAccount(); err != nil { return err } @@ -308,7 +308,7 @@ func (t *e2eElection) setupElection(ed *vapi.ElectionDescription) error { // if the census is not defined yet, set up a new census that will have // nvotes voterAccounts and participants if ed.Census.RootHash == nil { - censusRoot, censusURI, err := t.setupCensus(ed.Census.Type, t.config.nvotes) + censusRoot, censusURI, err := t.setupCensus(ed.Census.Type, t.config.nvotes, createAccounts) if err != nil { return err } @@ -341,7 +341,7 @@ func (t *e2eElection) setupElection(ed *vapi.ElectionDescription) error { return err } t.election = election - t.proofs, t.sikproofs = t.generateProofs(ed.Census.RootHash, ed.ElectionType.Anonymous, nil) + t.proofs, t.sikproofs = t.generateProofs(ed.Census.RootHash, ed.ElectionType.Anonymous, nil, createAccounts) return nil } @@ -366,7 +366,7 @@ func (t *e2eElection) setupElectionRaw(prc *models.Process) error { default: censusType := vapi.CensusTypeWeighted - censusRoot, censusURI, err := t.setupCensus(censusType, t.config.nvotes) + censusRoot, censusURI, err := t.setupCensus(censusType, t.config.nvotes, false) if err != nil { return err } @@ -388,7 +388,7 @@ func (t *e2eElection) setupElectionRaw(prc *models.Process) error { t.election = election prc.ProcessId = t.election.ElectionID - t.proofs, t.sikproofs = t.generateProofs(prc.CensusRoot, prc.EnvelopeType.Anonymous, csp) + t.proofs, t.sikproofs = t.generateProofs(prc.CensusRoot, prc.EnvelopeType.Anonymous, csp, false) return nil } diff --git a/cmd/end2endtest/lifecycle.go b/cmd/end2endtest/lifecycle.go index 97efc31ae..e7638d2eb 100644 --- a/cmd/end2endtest/lifecycle.go +++ b/cmd/end2endtest/lifecycle.go @@ -45,7 +45,7 @@ func (t *E2ELifecycleElection) Setup(api *apiclient.HTTPclient, c *config) error ed.d.ElectionType.Interruptible = ed.interruptible ed.d.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.elections[i].setupElection(ed.d); err != nil { + if err := t.elections[i].setupElection(ed.d, false); err != nil { log.Errorf("failed to setup election %d: %v", i, err) } } diff --git a/cmd/end2endtest/overwrite.go b/cmd/end2endtest/overwrite.go index c9adb21c8..27f032ab9 100644 --- a/cmd/end2endtest/overwrite.go +++ b/cmd/end2endtest/overwrite.go @@ -38,7 +38,7 @@ func (t *E2EOverwriteElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 2} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed); err != nil { + if err := t.setupElection(ed, false); err != nil { return err } diff --git a/cmd/end2endtest/plaintext.go b/cmd/end2endtest/plaintext.go index 2271fd8e5..23d10fc6e 100644 --- a/cmd/end2endtest/plaintext.go +++ b/cmd/end2endtest/plaintext.go @@ -35,7 +35,7 @@ func (t *E2EPlaintextElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed); err != nil { + if err := t.setupElection(ed, false); err != nil { return err } diff --git a/cmd/end2endtest/tempsiks.go b/cmd/end2endtest/tempsiks.go new file mode 100644 index 000000000..cf907bc4c --- /dev/null +++ b/cmd/end2endtest/tempsiks.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "fmt" + "math/big" + "os" + "time" + + vapi "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/apiclient" + "go.vocdoni.io/dvote/log" +) + +func init() { + ops["tempsiks"] = operation{ + test: &E2ETempSIKs{}, + description: "Performs a test of anonymous election with temporal SIKS", + example: os.Args[0] + " --operation=tempsiks --votes=1000", + } +} + +var _ VochainTest = (*E2ETempSIKs)(nil) + +type E2ETempSIKs struct { + e2eElection +} + +func (t *E2ETempSIKs) Setup(api *apiclient.HTTPclient, c *config) error { + t.api = api + t.config = c + + ed := newTestElectionDescription() + ed.ElectionType = vapi.ElectionType{ + Autostart: true, + Interruptible: true, + Anonymous: true, + } + ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} + ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeZKWeighted} + ed.TempSIKs = true + + if err := t.setupElection(ed, false); err != nil { + return err + } + log.Debugf("election details: %+v", *t.election) + return nil +} + +func (*E2ETempSIKs) Teardown() error { + // nothing to do here + return nil +} + +func (t *E2ETempSIKs) Run() error { + // Send the votes (parallelized) + startTime := time.Now() + + // register temporal SIKs + for i, acc := range t.voterAccounts { + log.Infow(fmt.Sprintf("register temporal sik %d/%d", i, t.config.nvotes), + "address", acc.AddressString()) + pKey := acc.PrivateKey() + client := t.api.Clone(pKey.String()) + hash, err := client.RegisterSIKForVote(t.election.ElectionID, t.proofs[acc.AddressString()], nil) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := client.WaitUntilTxIsMined(ctx, hash); err != nil { + return err + } + t.sikproofs[acc.AddressString()], err = client.GenSIKProof() + if err != nil { + return err + } + } + + log.Infow("enqueuing votes", "n", len(t.voterAccounts), "election", t.election.ElectionID) + votes := []*apiclient.VoteData{} + for i, acct := range t.voterAccounts { + votes = append(votes, &apiclient.VoteData{ + ElectionID: t.election.ElectionID, + ProofMkTree: t.proofs[acct.AddressString()], + ProofSIKTree: t.sikproofs[acct.AddressString()], + Choices: []int{i % 2}, + VoterAccount: acct, + VoteWeight: big.NewInt(defaultWeight / 2), + }) + } + t.sendVotes(votes) + + log.Infow("votes submitted successfully", + "n", len(t.voterAccounts), "time", time.Since(startTime), + "vps", int(float64(len(t.voterAccounts))/time.Since(startTime).Seconds())) + + if err := t.verifyVoteCount(t.config.nvotes); err != nil { + return err + } + + elres, err := t.endElectionAndFetchResults() + if err != nil { + return err + } + + log.Info("check if temporal SIKs are purged") + for _, acc := range t.voterAccounts { + pKey := acc.PrivateKey() + client := t.api.Clone(pKey.String()) + if valid, err := client.ValidSIK(); err == nil || valid { + return fmt.Errorf("sik expected to be removed") + } + } + + log.Infof("election %s status is RESULTS", t.election.ElectionID.String()) + log.Infof("election results: %v", elres.Results) + + return nil +} diff --git a/cmd/end2endtest/zkweighted.go b/cmd/end2endtest/zkweighted.go index 7487b232d..02529e93d 100644 --- a/cmd/end2endtest/zkweighted.go +++ b/cmd/end2endtest/zkweighted.go @@ -37,7 +37,7 @@ func (t *E2EAnonElection) Setup(api *apiclient.HTTPclient, c *config) error { ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeZKWeighted} - if err := t.setupElection(ed); err != nil { + if err := t.setupElection(ed, true); err != nil { return err } log.Debugf("election details: %+v", *t.election) diff --git a/vochain/ist/ist.go b/vochain/ist/ist.go index f11ade49a..7fcadd2b5 100644 --- a/vochain/ist/ist.go +++ b/vochain/ist/ist.go @@ -211,6 +211,18 @@ func (c *Controller) Commit(height uint32) error { return fmt.Errorf("cannot commit results for election %x: %w", action.ElectionID, err) } + // if the election is setted up with tempSIKs as true, purge the election + // related SIKs + process, err := c.state.Process(action.ElectionID, false) + if err != nil { + return fmt.Errorf("cannot get process: %w", err) + } + if process.GetTempSIKs() { + log.Infow("purge temporal siks", "pid", hex.EncodeToString(process.ProcessId)) + if err := c.state.PurgeSIKsByElection(process.ProcessId); err != nil { + return fmt.Errorf("cannot end election: %w", err) + } + } case ActionEndProcess: log.Debugw("end process", "height", height, "id", fmt.Sprintf("%x", id), "action", ActionsToString[action.ID]) if err := c.endElection(action.ElectionID); err != nil { @@ -253,7 +265,6 @@ func (c *Controller) endElection(electionID []byte) error { models.ProcessStatus_RESULTS: return nil } - // set the election to ended if err := c.state.SetProcessStatus(electionID, models.ProcessStatus_ENDED, true); err != nil { return fmt.Errorf("cannot end election: %w", err) diff --git a/vochain/state/sik.go b/vochain/state/sik.go index 7e18f3741..0f456628d 100644 --- a/vochain/state/sik.go +++ b/vochain/state/sik.go @@ -340,22 +340,23 @@ func (v *State) PurgeSIKsByElection(pid []byte) error { sikNoStateDB := v.NoState(false) // iterate to remove the assigned SIK to every address of this process and // also the relation between them - var iterErr error + toPurge := [][]byte{} if err := sikNoStateDB.Iterate(pid, func(address, _ []byte) bool { - // remove the SIK by the address - if iterErr = siksTree.Del(address); iterErr != nil { - return false - } - // remove the relation between process and address - if iterErr = sikNoStateDB.Delete(toPrefixKey(pid, address)); iterErr != nil { - return false - } + toPurge = append(toPurge, bytes.Clone(address)) return true }); err != nil { return fmt.Errorf("%w: %w", ErrSIKDelete, err) } - if iterErr != nil { - return fmt.Errorf("%w: %w", ErrSIKDelete, err) + for _, address := range toPurge { + // remove the SIK by the address + if err := siksTree.Del(address); err != nil { + return fmt.Errorf("%w: %w", ErrSIKDelete, err) + } + // remove the relation between process and address + if err := sikNoStateDB.Delete(toPrefixKey(pid, address)); err != nil { + return fmt.Errorf("%w: %w", ErrSIKDelete, err) + } + log.Infow("temporal SIK purged", "address", hex.EncodeToString(address)) } return nil } diff --git a/vochain/transaction/transaction.go b/vochain/transaction/transaction.go index ed07eac03..8487e25fb 100644 --- a/vochain/transaction/transaction.go +++ b/vochain/transaction/transaction.go @@ -7,6 +7,7 @@ import ( snarkTypes "github.com/vocdoni/go-snark/types" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/crypto/zk/circuit" + "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/vochain/ist" vstate "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" @@ -481,6 +482,7 @@ func (t *TransactionHandler) CheckTx(vtx *vochaintx.Tx, forCommit bool) (*Transa return nil, fmt.Errorf("registerSIKTx: %w", err) } if tempSIKs { + log.Infow("registering tempSIK", "address", txAddress.String()) if err := t.state.AssignSIKToElection(pid, txAddress); err != nil { return nil, fmt.Errorf("registerSIKTx: %w", err) } diff --git a/vochain/transaction_zk_test.go b/vochain/transaction_zk_test.go index d7bc2286f..bdaeb4e56 100644 --- a/vochain/transaction_zk_test.go +++ b/vochain/transaction_zk_test.go @@ -178,7 +178,7 @@ func TestMaxCensusSizeRegisterSIKTx(t *testing.T) { MaxCensusSize: 10, CensusOrigin: models.CensusOrigin_OFF_CHAIN_TREE_WEIGHTED, } - c.Check(app.State.AddProcess(process), qt.IsNil) + c.Assert(app.State.AddProcess(process), qt.IsNil) app.AdvanceTestBlock() encWeight := arbo.BigIntToBytes(arbo.HashFunctionPoseidon.Len(), testWeight) From 1171fe9b8f18bf4e4ccc36febaec7cb158fd563e Mon Sep 17 00:00:00 2001 From: "Lucas\\ Menendez" Date: Fri, 11 Aug 2023 14:01:27 +0200 Subject: [PATCH 02/15] changes on end2end tests: reverting almost all the tests to the previous API, splitting census proofs generation from siks proof generations. SIK related purges on election with tempSIKs moved to the top of the functions, trying to solve IST and State issue --- apiclient/account.go | 2 +- cmd/end2endtest/censusize.go | 2 +- cmd/end2endtest/dynamicensus.go | 2 +- cmd/end2endtest/encrypted.go | 2 +- cmd/end2endtest/helpers.go | 84 ++++++++++++--- cmd/end2endtest/lifecycle.go | 2 +- cmd/end2endtest/overwrite.go | 2 +- cmd/end2endtest/plaintext.go | 2 +- cmd/end2endtest/tempsiks.go | 120 --------------------- cmd/end2endtest/zkweighted.go | 2 +- test/siks_test.go | 168 +++++++++++++++++++++++++++++ vochain/ist/ist.go | 24 ++--- vochain/transaction/account_tx.go | 2 +- vochain/transaction/transaction.go | 8 +- 14 files changed, 262 insertions(+), 160 deletions(-) delete mode 100644 cmd/end2endtest/tempsiks.go create mode 100644 test/siks_test.go diff --git a/apiclient/account.go b/apiclient/account.go index 8a6a8cabb..08784d251 100644 --- a/apiclient/account.go +++ b/apiclient/account.go @@ -365,7 +365,7 @@ func (c *HTTPclient) RegisterSIKForVote(electionId types.HexBytes, proof *Census }, }) if err != nil { - return nil, fmt.Errorf("error enconsing RegisterSIKTx: %w", err) + return nil, fmt.Errorf("error enconding RegisterSIKTx: %w", err) } // sign it and send it hash, _, err := c.SignAndSendTx(stx) diff --git a/cmd/end2endtest/censusize.go b/cmd/end2endtest/censusize.go index dad8a784f..48fc60e28 100644 --- a/cmd/end2endtest/censusize.go +++ b/cmd/end2endtest/censusize.go @@ -41,7 +41,7 @@ func (t *E2EMaxCensusSizeElection) Setup(api *apiclient.HTTPclient, c *config) e Size: uint64(t.config.nvotes - 1), } - if err := t.setupElection(ed, false); err != nil { + if err := t.setupElection(ed); err != nil { return err } diff --git a/cmd/end2endtest/dynamicensus.go b/cmd/end2endtest/dynamicensus.go index 15b8262f4..d5b4b615a 100644 --- a/cmd/end2endtest/dynamicensus.go +++ b/cmd/end2endtest/dynamicensus.go @@ -62,7 +62,7 @@ func (t *E2EDynamicensusElection) Setup(api *apiclient.HTTPclient, c *config) er } // set up the election with the custom census created - if err := t.elections[i].setupElection(ed.d, false); err != nil { + if err := t.elections[i].setupElection(ed.d); err != nil { return err } log.Debugf("election detail: %+v", *t.elections[i].election) diff --git a/cmd/end2endtest/encrypted.go b/cmd/end2endtest/encrypted.go index e6dd77997..526a7dc01 100644 --- a/cmd/end2endtest/encrypted.go +++ b/cmd/end2endtest/encrypted.go @@ -36,7 +36,7 @@ func (t *E2EEncryptedElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed, false); err != nil { + if err := t.setupElection(ed); err != nil { return err } diff --git a/cmd/end2endtest/helpers.go b/cmd/end2endtest/helpers.go index a00fb879e..e9abb8365 100644 --- a/cmd/end2endtest/helpers.go +++ b/cmd/end2endtest/helpers.go @@ -167,13 +167,11 @@ func (t *e2eElection) waitUntilElectionStarts(electionID types.HexBytes) (*vapi. return election, nil } -func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool, csp *ethereum.SignKeys, generateSIKProofs bool) (map[string]*apiclient.CensusProof, map[string]*apiclient.CensusProof) { +func (t *e2eElection) generateSIKProofs(root types.HexBytes) map[string]*apiclient.CensusProof { type voterProof struct { - proof *apiclient.CensusProof sikproof *apiclient.CensusProof address string } - proofs := make(map[string]*apiclient.CensusProof, len(t.voterAccounts)) sikProofs := make(map[string]*apiclient.CensusProof, len(t.voterAccounts)) proofCh := make(chan *voterProof) stopProofs := make(chan bool) @@ -181,7 +179,6 @@ func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool for { select { case p := <-proofCh: - proofs[p.address] = p.proof sikProofs[p.address] = p.sikproof case <-stopProofs: return @@ -191,7 +188,61 @@ func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { defer wg.Done() - log.Infof("generating %d voting proofs", len(accounts)) + log.Infof("generating %d sik proofs", len(accounts)) + for _, acc := range accounts { + voterProof := &voterProof{address: acc.Address().Hex()} + + var err error + voterPrivKey := acc.PrivateKey() + voterApi := t.api.Clone(voterPrivKey.String()) + voterProof.sikproof, err = voterApi.GenSIKProof() + if err != nil { + log.Warn(err) + } + proofCh <- voterProof + } + } + + pcount := t.config.nvotes / t.config.parallelCount + var wg sync.WaitGroup + for i := 0; i < len(t.voterAccounts); i += pcount { + end := i + pcount + if end > len(t.voterAccounts) { + end = len(t.voterAccounts) + } + wg.Add(1) + go addNaccounts(t.voterAccounts[i:end], &wg) + } + + wg.Wait() + log.Debugf("%d/%d sik proofs generated successfully", len(sikProofs), len(t.voterAccounts)) + stopProofs <- true + + return sikProofs +} + +func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool, csp *ethereum.SignKeys) map[string]*apiclient.CensusProof { + type voterProof struct { + proof *apiclient.CensusProof + address string + } + proofs := make(map[string]*apiclient.CensusProof, len(t.voterAccounts)) + proofCh := make(chan *voterProof) + stopProofs := make(chan bool) + go func() { + for { + select { + case p := <-proofCh: + proofs[p.address] = p.proof + case <-stopProofs: + return + } + } + }() + + addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { + defer wg.Done() + log.Infof("generating %d census proofs", len(accounts)) for _, acc := range accounts { voterProof := &voterProof{address: acc.Address().Hex()} @@ -205,9 +256,6 @@ func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool if err != nil { log.Warn(err) } - if generateSIKProofs { - voterProof.sikproof, err = voterApi.GenSIKProof() - } } if err != nil { log.Warn(err) @@ -232,10 +280,10 @@ func (t *e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool } wg.Wait() - log.Debugf("%d/%d voting proofs generated successfully", len(proofs), len(t.voterAccounts)) + log.Debugf("%d/%d census proofs generated successfully", len(proofs), len(t.voterAccounts)) stopProofs <- true - return proofs, sikProofs + return proofs } func (t *e2eElection) setupAccount() error { @@ -261,7 +309,7 @@ func (t *e2eElection) setupCensus(censusType string, nAcct int, createAccounts b t.voterAccounts = append(t.voterAccounts, ethereum.NewSignKeysBatch(nAcct)...) // Register the accounts in the vochain if is required - if createAccounts { + if censusType == vapi.CensusTypeZKWeighted && createAccounts { for _, acc := range t.voterAccounts { pKey := acc.PrivateKey() if _, _, err := t.createAccount(pKey.String()); err != nil && @@ -300,7 +348,7 @@ func (t *e2eElection) setupCensus(censusType string, nAcct int, createAccounts b return censusRoot, censusURI, nil } -func (t *e2eElection) setupElection(ed *vapi.ElectionDescription, createAccounts bool) error { +func (t *e2eElection) setupElection(ed *vapi.ElectionDescription) error { if err := t.setupAccount(); err != nil { return err } @@ -308,7 +356,7 @@ func (t *e2eElection) setupElection(ed *vapi.ElectionDescription, createAccounts // if the census is not defined yet, set up a new census that will have // nvotes voterAccounts and participants if ed.Census.RootHash == nil { - censusRoot, censusURI, err := t.setupCensus(ed.Census.Type, t.config.nvotes, createAccounts) + censusRoot, censusURI, err := t.setupCensus(ed.Census.Type, t.config.nvotes, true) if err != nil { return err } @@ -341,7 +389,10 @@ func (t *e2eElection) setupElection(ed *vapi.ElectionDescription, createAccounts return err } t.election = election - t.proofs, t.sikproofs = t.generateProofs(ed.Census.RootHash, ed.ElectionType.Anonymous, nil, createAccounts) + t.proofs = t.generateProofs(ed.Census.RootHash, ed.ElectionType.Anonymous, nil) + if ed.ElectionType.Anonymous && !ed.TempSIKs { + t.sikproofs = t.generateSIKProofs(ed.Census.RootHash) + } return nil } @@ -388,7 +439,10 @@ func (t *e2eElection) setupElectionRaw(prc *models.Process) error { t.election = election prc.ProcessId = t.election.ElectionID - t.proofs, t.sikproofs = t.generateProofs(prc.CensusRoot, prc.EnvelopeType.Anonymous, csp, false) + t.proofs = t.generateProofs(prc.CensusRoot, prc.EnvelopeType.Anonymous, csp) + if prc.EnvelopeType.Anonymous { + t.sikproofs = t.generateSIKProofs(prc.CensusRoot) + } return nil } diff --git a/cmd/end2endtest/lifecycle.go b/cmd/end2endtest/lifecycle.go index e7638d2eb..97efc31ae 100644 --- a/cmd/end2endtest/lifecycle.go +++ b/cmd/end2endtest/lifecycle.go @@ -45,7 +45,7 @@ func (t *E2ELifecycleElection) Setup(api *apiclient.HTTPclient, c *config) error ed.d.ElectionType.Interruptible = ed.interruptible ed.d.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.elections[i].setupElection(ed.d, false); err != nil { + if err := t.elections[i].setupElection(ed.d); err != nil { log.Errorf("failed to setup election %d: %v", i, err) } } diff --git a/cmd/end2endtest/overwrite.go b/cmd/end2endtest/overwrite.go index 27f032ab9..c9adb21c8 100644 --- a/cmd/end2endtest/overwrite.go +++ b/cmd/end2endtest/overwrite.go @@ -38,7 +38,7 @@ func (t *E2EOverwriteElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 2} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed, false); err != nil { + if err := t.setupElection(ed); err != nil { return err } diff --git a/cmd/end2endtest/plaintext.go b/cmd/end2endtest/plaintext.go index 23d10fc6e..2271fd8e5 100644 --- a/cmd/end2endtest/plaintext.go +++ b/cmd/end2endtest/plaintext.go @@ -35,7 +35,7 @@ func (t *E2EPlaintextElection) Setup(api *apiclient.HTTPclient, c *config) error ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - if err := t.setupElection(ed, false); err != nil { + if err := t.setupElection(ed); err != nil { return err } diff --git a/cmd/end2endtest/tempsiks.go b/cmd/end2endtest/tempsiks.go deleted file mode 100644 index cf907bc4c..000000000 --- a/cmd/end2endtest/tempsiks.go +++ /dev/null @@ -1,120 +0,0 @@ -package main - -import ( - "context" - "fmt" - "math/big" - "os" - "time" - - vapi "go.vocdoni.io/dvote/api" - "go.vocdoni.io/dvote/apiclient" - "go.vocdoni.io/dvote/log" -) - -func init() { - ops["tempsiks"] = operation{ - test: &E2ETempSIKs{}, - description: "Performs a test of anonymous election with temporal SIKS", - example: os.Args[0] + " --operation=tempsiks --votes=1000", - } -} - -var _ VochainTest = (*E2ETempSIKs)(nil) - -type E2ETempSIKs struct { - e2eElection -} - -func (t *E2ETempSIKs) Setup(api *apiclient.HTTPclient, c *config) error { - t.api = api - t.config = c - - ed := newTestElectionDescription() - ed.ElectionType = vapi.ElectionType{ - Autostart: true, - Interruptible: true, - Anonymous: true, - } - ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} - ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeZKWeighted} - ed.TempSIKs = true - - if err := t.setupElection(ed, false); err != nil { - return err - } - log.Debugf("election details: %+v", *t.election) - return nil -} - -func (*E2ETempSIKs) Teardown() error { - // nothing to do here - return nil -} - -func (t *E2ETempSIKs) Run() error { - // Send the votes (parallelized) - startTime := time.Now() - - // register temporal SIKs - for i, acc := range t.voterAccounts { - log.Infow(fmt.Sprintf("register temporal sik %d/%d", i, t.config.nvotes), - "address", acc.AddressString()) - pKey := acc.PrivateKey() - client := t.api.Clone(pKey.String()) - hash, err := client.RegisterSIKForVote(t.election.ElectionID, t.proofs[acc.AddressString()], nil) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - if _, err := client.WaitUntilTxIsMined(ctx, hash); err != nil { - return err - } - t.sikproofs[acc.AddressString()], err = client.GenSIKProof() - if err != nil { - return err - } - } - - log.Infow("enqueuing votes", "n", len(t.voterAccounts), "election", t.election.ElectionID) - votes := []*apiclient.VoteData{} - for i, acct := range t.voterAccounts { - votes = append(votes, &apiclient.VoteData{ - ElectionID: t.election.ElectionID, - ProofMkTree: t.proofs[acct.AddressString()], - ProofSIKTree: t.sikproofs[acct.AddressString()], - Choices: []int{i % 2}, - VoterAccount: acct, - VoteWeight: big.NewInt(defaultWeight / 2), - }) - } - t.sendVotes(votes) - - log.Infow("votes submitted successfully", - "n", len(t.voterAccounts), "time", time.Since(startTime), - "vps", int(float64(len(t.voterAccounts))/time.Since(startTime).Seconds())) - - if err := t.verifyVoteCount(t.config.nvotes); err != nil { - return err - } - - elres, err := t.endElectionAndFetchResults() - if err != nil { - return err - } - - log.Info("check if temporal SIKs are purged") - for _, acc := range t.voterAccounts { - pKey := acc.PrivateKey() - client := t.api.Clone(pKey.String()) - if valid, err := client.ValidSIK(); err == nil || valid { - return fmt.Errorf("sik expected to be removed") - } - } - - log.Infof("election %s status is RESULTS", t.election.ElectionID.String()) - log.Infof("election results: %v", elres.Results) - - return nil -} diff --git a/cmd/end2endtest/zkweighted.go b/cmd/end2endtest/zkweighted.go index 02529e93d..7487b232d 100644 --- a/cmd/end2endtest/zkweighted.go +++ b/cmd/end2endtest/zkweighted.go @@ -37,7 +37,7 @@ func (t *E2EAnonElection) Setup(api *apiclient.HTTPclient, c *config) error { ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeZKWeighted} - if err := t.setupElection(ed, true); err != nil { + if err := t.setupElection(ed); err != nil { return err } log.Debugf("election details: %+v", *t.election) diff --git a/test/siks_test.go b/test/siks_test.go new file mode 100644 index 000000000..81bbbb22c --- /dev/null +++ b/test/siks_test.go @@ -0,0 +1,168 @@ +package test + +import ( + "encoding/json" + "math/big" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/google/uuid" + "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/crypto/ethereum" + "go.vocdoni.io/dvote/data/ipfs" + "go.vocdoni.io/dvote/test/testcommon" + "go.vocdoni.io/dvote/test/testcommon/testutil" + "go.vocdoni.io/dvote/types" + "go.vocdoni.io/dvote/util" + "go.vocdoni.io/dvote/vochain" + "go.vocdoni.io/dvote/vochain/ist" + "go.vocdoni.io/proto/build/go/models" + "google.golang.org/protobuf/proto" +) + +func TestTempSIKs(t *testing.T) { + c := qt.New(t) + + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + api.SIKHandler, + ) + token1 := uuid.New() + client := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 1) + + // create a new account + admin := ethereum.SignKeys{} + c.Assert(admin.Generate(), qt.IsNil) + + // metdata + meta := &api.AccountMetadata{ + Version: "1.0", + } + metaData, err := json.Marshal(meta) + c.Assert(err, qt.IsNil) + + // transaction + faucetPkg, err := vochain.GenerateFaucetPackage(server.Account, admin.Address(), 10000) + c.Assert(err, qt.IsNil) + stx := models.SignedTx{} + infoURI := server.Storage.URIprefix() + ipfs.CalculateCIDv1json(metaData) + sik, err := admin.AccountSIK(nil) + c.Assert(err, qt.IsNil) + stx.Tx, err = proto.Marshal(&models.Tx{Payload: &models.Tx_SetAccount{ + SetAccount: &models.SetAccountTx{ + Txtype: models.TxType_CREATE_ACCOUNT, + Nonce: new(uint32), + InfoURI: &infoURI, + Account: admin.Address().Bytes(), + FaucetPackage: faucetPkg, + SIK: sik, + }, + }}) + c.Assert(err, qt.IsNil) + stx.Signature, err = admin.SignVocdoniTx(stx.Tx, server.VochainAPP.ChainID()) + c.Assert(err, qt.IsNil) + stxb, err := proto.Marshal(&stx) + c.Assert(err, qt.IsNil) + // send the transaction and metadata + accSet := api.AccountSet{ + Metadata: metaData, + TxPayload: stxb, + } + resp, code := client.Request("POST", &accSet, "accounts") + c.Assert(code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 2) + + // create a new census + resp, code = client.Request("POST", nil, "censuses", api.CensusTypeZKWeighted) + c.Assert(code, qt.Equals, 200) + censusData := &api.Census{} + c.Assert(json.Unmarshal(resp, censusData), qt.IsNil) + censusId := censusData.CensusID.String() + // add a voter key + voter := ethereum.NewSignKeys() + c.Assert(voter.Generate(), qt.IsNil) + cparts := api.CensusParticipants{Participants: []api.CensusParticipant{ + {Key: voter.Address().Bytes(), Weight: (*types.BigInt)(big.NewInt(int64(1)))}, + }} + _, code = client.Request("POST", &cparts, "censuses", censusId, "participants") + c.Assert(code, qt.Equals, 200) + // publish and get the root + resp, code = client.Request("POST", nil, "censuses", censusId, "publish") + c.Assert(code, qt.Equals, 200) + c.Assert(json.Unmarshal(resp, censusData), qt.IsNil) + c.Assert(censusData.CensusID, qt.IsNotNil) + censusRoot := censusData.CensusID + // create an anonymous election + metadataBytes, err := json.Marshal( + &api.ElectionMetadata{ + Title: map[string]string{"default": "test election"}, + Description: map[string]string{"default": "test election description"}, + Version: "1.0", + }) + c.Assert(err, qt.IsNil) + metadataURI := ipfs.CalculateCIDv1json(metadataBytes) + var electionId types.HexBytes = util.RandomBytes(types.ProcessIDsize) + tempSIKs := true + process := &models.Process{ + EntityId: admin.Address().Bytes(), + ProcessId: electionId, + StartBlock: 0, + BlockCount: 100, + Status: models.ProcessStatus_READY, + CensusRoot: censusRoot, + CensusOrigin: models.CensusOrigin_OFF_CHAIN_TREE_WEIGHTED, + CensusURI: &censusData.URI, + Mode: &models.ProcessMode{AutoStart: true, Interruptible: true}, + VoteOptions: &models.ProcessVoteOptions{MaxCount: 1, MaxValue: 1}, + EnvelopeType: &models.EnvelopeType{Anonymous: true}, + Metadata: &metadataURI, + MaxCensusSize: 1000, + TempSIKs: &tempSIKs, + } + c.Assert(server.VochainAPP.State.AddProcess(process), qt.IsNil) + + // Block 3 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 3) + // register voter temporal sik + voterSIK, err := voter.AccountSIK(nil) + c.Assert(err, qt.IsNil) + c.Assert(server.VochainAPP.State.SetAddressSIK(voter.Address(), voterSIK), qt.IsNil) + c.Assert(server.VochainAPP.State.IncreaseRegisterSIKCounter(electionId), qt.IsNil) + c.Assert(server.VochainAPP.State.AssignSIKToElection(electionId, voter.Address()), qt.IsNil) + + // Block 4 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 4) + // schedule election results and finishing election + c.Assert(server.VochainAPP.State.SetProcessStatus(electionId, models.ProcessStatus_ENDED, true), qt.IsNil) + c.Assert(server.VochainAPP.Istc.Schedule(5, electionId, ist.Action{ + ID: ist.ActionCommitResults, + ElectionID: electionId, + }), qt.IsNil) + + // Block 5 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 5) + // check that the election has ended + electionData := &api.Election{} + resp, code = client.Request("GET", nil, "elections", electionId.String()) + c.Assert(code, qt.Equals, 200) + c.Assert(json.Unmarshal(resp, electionData), qt.IsNil) + c.Assert(electionData.Status, qt.Equals, "ENDED") + // check that the voter sik has been deleted + _, err = server.VochainAPP.State.SIKFromAddress(voter.Address()) + c.Assert(err, qt.IsNotNil) +} diff --git a/vochain/ist/ist.go b/vochain/ist/ist.go index 7fcadd2b5..abc5562ca 100644 --- a/vochain/ist/ist.go +++ b/vochain/ist/ist.go @@ -191,6 +191,18 @@ func (c *Controller) Commit(height uint32) error { for id, action := range actions { switch action.ID { case ActionCommitResults: + // if the election is setted up with tempSIKs as true, purge the election + // related SIKs + process, err := c.state.Process(action.ElectionID, false) + if err != nil { + return fmt.Errorf("cannot get process: %w", err) + } + if process.GetTempSIKs() { + log.Infow("purge temporal siks", "pid", hex.EncodeToString(process.ProcessId)) + if err := c.state.PurgeSIKsByElection(process.ProcessId); err != nil { + return fmt.Errorf("cannot purge temp SIKs after election ends: %w", err) + } + } log.Debugw("commit results", "height", height, "id", fmt.Sprintf("%x", id), "action", ActionsToString[action.ID]) var r *results.Results if !c.checkRevealedKeys(action.ElectionID) { @@ -211,18 +223,6 @@ func (c *Controller) Commit(height uint32) error { return fmt.Errorf("cannot commit results for election %x: %w", action.ElectionID, err) } - // if the election is setted up with tempSIKs as true, purge the election - // related SIKs - process, err := c.state.Process(action.ElectionID, false) - if err != nil { - return fmt.Errorf("cannot get process: %w", err) - } - if process.GetTempSIKs() { - log.Infow("purge temporal siks", "pid", hex.EncodeToString(process.ProcessId)) - if err := c.state.PurgeSIKsByElection(process.ProcessId); err != nil { - return fmt.Errorf("cannot end election: %w", err) - } - } case ActionEndProcess: log.Debugw("end process", "height", height, "id", fmt.Sprintf("%x", id), "action", ActionsToString[action.ID]) if err := c.endElection(action.ElectionID); err != nil { diff --git a/vochain/transaction/account_tx.go b/vochain/transaction/account_tx.go index aa747cd02..0660126ef 100644 --- a/vochain/transaction/account_tx.go +++ b/vochain/transaction/account_tx.go @@ -392,7 +392,7 @@ func (t *TransactionHandler) RegisterSIKTxCheck(vtx *vochaintx.Tx) (common.Addre return common.Address{}, nil, nil, false, fmt.Errorf("error verifying the proof: %w", err) } if !valid { - return common.Address{}, nil, nil, false, fmt.Errorf("proof not valid") + return common.Address{}, nil, nil, false, fmt.Errorf("proof not valid: %x", process.CensusRoot) } return txAddress, newSIK, pid, process.GetTempSIKs(), nil } diff --git a/vochain/transaction/transaction.go b/vochain/transaction/transaction.go index 8487e25fb..e128897d0 100644 --- a/vochain/transaction/transaction.go +++ b/vochain/transaction/transaction.go @@ -158,6 +158,10 @@ func (t *TransactionHandler) CheckTx(vtx *vochaintx.Tx, forCommit bool) (*Transa return nil, fmt.Errorf("setProcessStatus: %s", err) } if tx.GetStatus() == models.ProcessStatus_ENDED { + // purge RegisterSIKTx counter if it exists + if err := t.state.PurgeRegisterSIK(tx.ProcessId); err != nil { + return nil, fmt.Errorf("setProcessStatus: cannot purge RegisterSIKTx counter: %w", err) + } // schedule results computations on the ISTC if err := t.istc.Schedule(t.state.CurrentHeight()+1, tx.ProcessId, ist.Action{ ID: ist.ActionCommitResults, @@ -165,10 +169,6 @@ func (t *TransactionHandler) CheckTx(vtx *vochaintx.Tx, forCommit bool) (*Transa }); err != nil { return nil, fmt.Errorf("setProcessStatus: cannot schedule end process: %w", err) } - // purge RegisterSIKTx counter if it exists - if err := t.state.PurgeRegisterSIK(tx.ProcessId); err != nil { - return nil, fmt.Errorf("setProcessStatus: cannot purge RegisterSIKTx counter: %w", err) - } } case models.TxType_SET_PROCESS_CENSUS: From 5b54de15d320296ddbc50717f24cd6ac9b639b0e Mon Sep 17 00:00:00 2001 From: p4u Date: Tue, 22 Aug 2023 00:50:05 +0200 Subject: [PATCH 03/15] add NoState test --- test/siks_test.go | 2 +- vochain/state/state_test.go | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test/siks_test.go b/test/siks_test.go index 81bbbb22c..73844922f 100644 --- a/test/siks_test.go +++ b/test/siks_test.go @@ -164,5 +164,5 @@ func TestTempSIKs(t *testing.T) { c.Assert(electionData.Status, qt.Equals, "ENDED") // check that the voter sik has been deleted _, err = server.VochainAPP.State.SIKFromAddress(voter.Address()) - c.Assert(err, qt.IsNotNil) + c.Assert(err, qt.IsNotNil, qt.Commentf("voter sik should have been deleted")) } diff --git a/vochain/state/state_test.go b/vochain/state/state_test.go index 852581527..5eaed6ffa 100644 --- a/vochain/state/state_test.go +++ b/vochain/state/state_test.go @@ -1,6 +1,7 @@ package state import ( + "bytes" "fmt" "runtime" "testing" @@ -392,3 +393,53 @@ func TestStateSetGetTxCostByTxType(t *testing.T) { qt.Assert(t, err, qt.IsNil) qt.Assert(t, cost, qt.Equals, uint64(100)) } + +func TestNoState(t *testing.T) { + s, err := NewState(db.TypePebble, "") + qt.Assert(t, err, qt.IsNil) + defer func() { + _ = s.Close() + }() + + doBlock := func(height uint32, fn func()) []byte { + s.Rollback() + s.SetHeight(height) + fn() + h, err := s.Save() + qt.Assert(t, err, qt.IsNil) + return h + } + + ns := s.NoState(true) + + // commit first block + hash1 := doBlock(1, func() {}) + + // commit second block + hash2 := doBlock(2, func() { + err = ns.Set([]byte("foo"), []byte("bar")) + qt.Assert(t, err, qt.IsNil) + }) + + // compare hash1 and hash 2, root hash should be the same + qt.Assert(t, bytes.Equal(hash1, hash2), qt.IsTrue) + + // check that the is still in the nostate + value, err := ns.Get([]byte("foo")) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, value, qt.CmpEquals(), []byte("bar")) + + // commit third block + hash3 := doBlock(3, func() { + err = ns.Delete([]byte("foo")) + qt.Assert(t, err, qt.IsNil) + }) + + // compare hash2 and hash3, root hash should be the same + qt.Assert(t, bytes.Equal(hash2, hash3), qt.IsTrue) + + // check that the value is not in the nostate + _, err = ns.Get([]byte("foo")) + qt.Assert(t, err, qt.Equals, db.ErrKeyNotFound) + +} From 58f5afefb7725f25765cf5bfdb98bf5d9042a721 Mon Sep 17 00:00:00 2001 From: p4u Date: Thu, 24 Aug 2023 14:59:16 +0200 Subject: [PATCH 04/15] arbo: implement the Delete() method For the temporary SIK elections, we need Arbo to implement a Delete() mechanism. The current Delete method removes the key, leaf and also all the intermediary nodes that are not needed. Signed-off-by: p4u --- test/siks_test.go | 9 ++-- tree/arbo/tree.go | 105 ++++++++++++++++++++++++++++++++++++-- tree/arbo/tree_test.go | 60 ++++++++++++++++++++++ tree/tree.go | 4 +- vochain/apptest.go | 11 +++- vochain/ist/ist.go | 1 + vochain/state/sik_test.go | 2 +- vochain/state/utils.go | 4 +- 8 files changed, 182 insertions(+), 14 deletions(-) diff --git a/test/siks_test.go b/test/siks_test.go index 73844922f..889965e0b 100644 --- a/test/siks_test.go +++ b/test/siks_test.go @@ -153,16 +153,17 @@ func TestTempSIKs(t *testing.T) { ElectionID: electionId, }), qt.IsNil) - // Block 5 + // Block 6 server.VochainAPP.AdvanceTestBlock() - waitUntilHeight(t, client, 5) + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, client, 6) // check that the election has ended electionData := &api.Election{} resp, code = client.Request("GET", nil, "elections", electionId.String()) c.Assert(code, qt.Equals, 200) c.Assert(json.Unmarshal(resp, electionData), qt.IsNil) - c.Assert(electionData.Status, qt.Equals, "ENDED") + c.Assert(electionData.Status, qt.Equals, "RESULTS") // check that the voter sik has been deleted _, err = server.VochainAPP.State.SIKFromAddress(voter.Address()) - c.Assert(err, qt.IsNotNil, qt.Commentf("voter sik should have been deleted")) + c.Assert(err, qt.IsNotNil, qt.Commentf("voter sik should have been deleted: %x", voter.Address())) } diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 2558e9c50..61c0e687d 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -618,7 +618,6 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt if err != nil { return nil, err } - if err := wTx.Set(leafKey, leafValue); err != nil { return nil, err } @@ -920,10 +919,110 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { } // store root to db - if err := t.setRoot(wTx, root); err != nil { + return t.setRoot(wTx, root) +} + +// Delete removes the key from the Tree. +func (t *Tree) Delete(k []byte) error { + wTx := t.db.WriteTx() + defer wTx.Discard() + + if err := t.DeleteWithTx(wTx, k); err != nil { return err } - return nil + + return wTx.Commit() +} + +// DeleteWithTx does the same as the Delete method, but allows passing the +// db.WriteTx that is used. The db.WriteTx will not be committed inside this +// method. +func (t *Tree) DeleteWithTx(wTx db.WriteTx, k []byte) error { + t.Lock() + defer t.Unlock() + + if !t.editable() { + return ErrSnapshotNotEditable + } + return t.deleteWithTx(wTx, k) +} + +// deleteWithTx deletes a key-value pair from the tree. +func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { + root, err := t.RootWithTx(wTx) + if err != nil { + return err + } + + // Navigate down the tree to find the key and collect siblings. + var siblings [][]byte + path := getPath(t.maxLevels, k) + _, _, siblings, err = t.down(wTx, k, root, siblings, path, 0, true) + if err != nil { + if err == ErrKeyNotFound { + // Key not found, nothing to delete. + return nil + } + return err + } + + // Navigate back up the tree, updating the intermediate nodes. + if len(siblings) == 0 { + // The tree is empty after deletion. + return t.setRoot(wTx, t.emptyHash) + } + newRoot, err := t.upAfterDelete(wTx, t.emptyHash, siblings, path, len(siblings)-1) + if err != nil { + return err + } + + // Update the root of the tree. + if err := t.setRoot(wTx, newRoot); err != nil { + return err + } + + // Update the number of leaves. + return t.decNLeafs(wTx, 1) +} + +// upAfterDelete navigates back up the tree after a delete operation, updating +// the intermediate nodes and potentially removing nodes that are no longer needed. +func (t *Tree) upAfterDelete(wTx db.WriteTx, key []byte, siblings [][]byte, path []bool, currLvl int) ([]byte, error) { + if currLvl < 0 { + return key, nil + } + + var k, v []byte + var err error + if path[currLvl] { + k, v, err = t.newIntermediate(siblings[currLvl], key) + } else { + k, v, err = t.newIntermediate(key, siblings[currLvl]) + } + if err != nil { + return nil, err + } + + // If both children are empty, remove the intermediate node. + if bytes.Equal(siblings[currLvl], t.emptyHash) && bytes.Equal(key, t.emptyHash) { + return t.emptyHash, nil + } + + // Otherwise, store the updated intermediate node. + if err = wTx.Set(k, v); err != nil { + return nil, err + } + + return t.upAfterDelete(wTx, k, siblings, path, currLvl-1) +} + +// decNLeafs decreases the number of leaves in the tree. +func (t *Tree) decNLeafs(wTx db.WriteTx, n int) error { + nLeafs, err := t.GetNLeafsWithTx(wTx) + if err != nil { + return err + } + return t.setNLeafs(wTx, nLeafs-n) } // GenProof generates a MerkleTree proof for the given key. The leaf value is diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 7cff9d1d0..89bc48556 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -903,6 +903,66 @@ func TestKeyLenBiggerThan32(t *testing.T) { c.Assert(err, qt.IsNil) } +func TestDelete(t *testing.T) { + c := qt.New(t) + database := metadb.NewTest(t) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, + HashFunction: HashFunctionPoseidon}) + c.Assert(err, qt.IsNil) + + bLen := 32 + // Add multiple key/value pairs to the tree + keys := [][]byte{ + BigIntToBytes(bLen, big.NewInt(int64(1))), + BigIntToBytes(bLen, big.NewInt(int64(2))), + BigIntToBytes(bLen, big.NewInt(int64(3))), + BigIntToBytes(bLen, big.NewInt(int64(4))), + } + values := [][]byte{ + BigIntToBytes(bLen, big.NewInt(int64(10))), + BigIntToBytes(bLen, big.NewInt(int64(20))), + BigIntToBytes(bLen, big.NewInt(int64(30))), + BigIntToBytes(bLen, big.NewInt(int64(40))), + } + for i, key := range keys { + err := tree.Add(key, values[i]) + c.Assert(err, qt.IsNil) + } + + originalRoot, err := tree.Root() + c.Assert(err, qt.IsNil) + + // Delete a key from the tree + err = tree.Delete(keys[2]) + c.Assert(err, qt.IsNil) + + // Check if the key was deleted + _, _, err = tree.Get(keys[2]) + c.Assert(err, qt.Equals, ErrKeyNotFound) + + // Check the root has changed + afterDeleteRoot, err := tree.Root() + c.Assert(err, qt.IsNil) + c.Check(afterDeleteRoot, qt.Not(qt.DeepEquals), originalRoot) + + // Check if other keys are unaffected + for i, key := range keys { + if i == 2 { + continue // skip the deleted key + } + _, value, err := tree.Get(key) + c.Assert(err, qt.IsNil) + c.Check(value, qt.DeepEquals, values[i]) + } + + // Check the root hash is the same after re-adding the deleted key + err = tree.Add(keys[2], values[2]) // re-add the deleted key + c.Assert(err, qt.IsNil) + afterAddRoot, err := tree.Root() + c.Assert(err, qt.IsNil) + c.Check(afterAddRoot, qt.DeepEquals, originalRoot) +} + func BenchmarkAdd(b *testing.B) { bLen := 32 // for both Poseidon & Sha256 // prepare inputs diff --git a/tree/tree.go b/tree/tree.go index cd8c44dfe..79e5aacb1 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -127,8 +127,8 @@ func (t *Tree) Del(wTx db.WriteTx, key []byte) error { wTx = t.DB().WriteTx() defer wTx.Discard() } - if err := wTx.Delete(key); err != nil { - return err + if err := t.tree.DeleteWithTx(wTx, key); err != nil { + return fmt.Errorf("could not remove key: %w", err) } if !givenTx { return wTx.Commit() diff --git a/vochain/apptest.go b/vochain/apptest.go index f9c002452..e42d56bc9 100644 --- a/vochain/apptest.go +++ b/vochain/apptest.go @@ -97,13 +97,20 @@ func (app *BaseApplication) SetTestingMethods() { // AdvanceTestBlock commits the current state, ends the current block and starts a new one. // Advances the block height and timestamp. func (app *BaseApplication) AdvanceTestBlock() { + height := uint32(app.testMockBlockStore.Height()) + // execute internal state transition commit + if err := app.Istc.Commit(height); err != nil { + panic(err) + } + // finalize block + app.endBlock(time.Now(), height) + // save the state _, err := app.Commit(context.TODO(), nil) if err != nil { panic(err) } - app.endBlock(time.Now(), uint32(app.testMockBlockStore.Height())) - newHeight := app.testMockBlockStore.EndBlock() // The next block begins 50ms later + newHeight := app.testMockBlockStore.EndBlock() time.Sleep(time.Millisecond * 50) nextStartTime := time.Now() app.testMockBlockStore.NewBlock(newHeight) diff --git a/vochain/ist/ist.go b/vochain/ist/ist.go index abc5562ca..43e14f1e4 100644 --- a/vochain/ist/ist.go +++ b/vochain/ist/ist.go @@ -188,6 +188,7 @@ func (c *Controller) Commit(height uint32) error { if len(actions) == 0 { return nil } + for id, action := range actions { switch action.ID { case ActionCommitResults: diff --git a/vochain/state/sik_test.go b/vochain/state/sik_test.go index 6dfe2e99c..37ccd2fd0 100644 --- a/vochain/state/sik_test.go +++ b/vochain/state/sik_test.go @@ -165,7 +165,7 @@ func TestAssignSIKToElectionAndPurge(t *testing.T) { _, err = s.NoState(true).Get(key) c.Assert(err, qt.IsNotNil) _, err = s.SIKFromAddress(testAccount.Address()) - c.Assert(err, qt.IsNil) + c.Assert(err, qt.IsNotNil) } func Test_heightEncoding(t *testing.T) { diff --git a/vochain/state/utils.go b/vochain/state/utils.go index 6e49d203b..4501604ac 100644 --- a/vochain/state/utils.go +++ b/vochain/state/utils.go @@ -61,8 +61,8 @@ func printPrettierDelegates(delegates [][]byte) []string { // toPrefixKey helper function returns the encoded key of the provided one // prefixing it with the prefix provided. -func toPrefixKey(prefix, originaKey []byte) []byte { - return append([]byte(prefix), originaKey...) +func toPrefixKey(prefix, key []byte) []byte { + return append([]byte(prefix), key...) } // fromPrefixKey helper function returns the original key of the provided From b850224a92cc22bf232459004febeaee42f8f75b Mon Sep 17 00:00:00 2001 From: p4u Date: Mon, 28 Aug 2023 23:22:02 +0200 Subject: [PATCH 05/15] arbo: fix clean-up of empty nodes when going up Fix the upAfterDelete function and unify it with up(). So now also Update and Add methods are removing empty nodes. Add a test to check the disk size after some operations. Signed-off-by: p4u --- tree/arbo/addbatch_test.go | 16 ++--- tree/arbo/tree.go | 69 ++++++------------ tree/arbo/tree_test.go | 141 +++++++++++++++++++++++++++++++++---- 3 files changed, 159 insertions(+), 67 deletions(-) diff --git a/tree/arbo/addbatch_test.go b/tree/arbo/addbatch_test.go index 18c5b759d..2c21fc0c8 100644 --- a/tree/arbo/addbatch_test.go +++ b/tree/arbo/addbatch_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "math/big" + "os" "runtime" "sort" "testing" @@ -17,7 +18,7 @@ import ( "go.vocdoni.io/dvote/db/pebbledb" ) -var debug = false +var debug = os.Getenv("LOG_LEVEL") == "debug" func printTestContext(prefix string, nLeafs int, hashName, dbName string) { if debug { @@ -26,9 +27,9 @@ func printTestContext(prefix string, nLeafs int, hashName, dbName string) { } } -func printRes(name string, duration time.Duration) { +func printRes(name string, value interface{}) { if debug { - fmt.Printf("%s: %s \n", name, duration) + fmt.Printf("%s: %s \n", name, value) } } @@ -72,8 +73,7 @@ func TestAddBatchTreeEmpty(t *testing.T) { nLeafs := 1024 database := metadb.NewTest(t) - tree, err := NewTree(Config{database, 256, DefaultThresholdNLeafs, - HashFunctionPoseidon}) + tree, err := NewTree(Config{database, 256, DefaultThresholdNLeafs, HashFunctionPoseidon}) c.Assert(err, qt.IsNil) bLen := 32 @@ -94,8 +94,7 @@ func TestAddBatchTreeEmpty(t *testing.T) { time1 := time.Since(start) database2 := metadb.NewTest(t) - tree2, err := NewTree(Config{database2, 256, DefaultThresholdNLeafs, - HashFunctionPoseidon}) + tree2, err := NewTree(Config{database2, 256, DefaultThresholdNLeafs, HashFunctionPoseidon}) c.Assert(err, qt.IsNil) tree2.dbgInit() @@ -763,8 +762,7 @@ func benchAdd(t *testing.T, ks, vs [][]byte) { c := qt.New(t) database := metadb.NewTest(t) - tree, err := NewTree(Config{database, 256, DefaultThresholdNLeafs, - HashFunctionBlake2b}) + tree, err := NewTree(Config{database, 256, DefaultThresholdNLeafs, HashFunctionBlake2b}) c.Assert(err, qt.IsNil) start := time.Now() diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 61c0e687d..206ebcc15 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -153,7 +153,7 @@ func NewTreeWithTx(wTx db.WriteTx, cfg Config) (*Tree, error) { } return &t, nil } else if err != nil { - return nil, err + return nil, fmt.Errorf("unknown database error: %w", err) } return &t, nil } @@ -733,32 +733,40 @@ func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath, return siblings, nil } -// up goes up recursively updating the intermediate nodes -func (t *Tree) up(wTx db.WriteTx, key []byte, siblings [][]byte, path []bool, - currLvl, toLvl int) ([]byte, error) { +// up navigates back up the tree after a delete operation, updating +// the intermediate nodes and potentially removing nodes that are no longer needed. +func (t *Tree) up(wTx db.WriteTx, key []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) { + if currLvl < 0 { + return key, nil + } + var k, v []byte var err error if path[currLvl+toLvl] { k, v, err = t.newIntermediate(siblings[currLvl], key) - if err != nil { - return nil, err - } } else { k, v, err = t.newIntermediate(key, siblings[currLvl]) - if err != nil { + } + if err != nil { + return nil, err + } + + // If both children are empty, remove the intermediate node and its children. + if bytes.Equal(k, t.emptyHash) && bytes.Equal(v, t.emptyHash) { + if err := wTx.Delete(key); err != nil { return nil, err } + if err := wTx.Delete(siblings[currLvl]); err != nil { + return nil, err + } + return t.emptyHash, wTx.Set(k, t.emptyHash) } - // store k-v to db + + // Otherwise, store the updated intermediate node. if err = wTx.Set(k, v); err != nil { return nil, err } - if currLvl == 0 { - // reached the root - return k, nil - } - return t.up(wTx, k, siblings, path, currLvl-1, toLvl) } @@ -971,7 +979,7 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { // The tree is empty after deletion. return t.setRoot(wTx, t.emptyHash) } - newRoot, err := t.upAfterDelete(wTx, t.emptyHash, siblings, path, len(siblings)-1) + newRoot, err := t.up(wTx, t.emptyHash, siblings, path, len(siblings)-1, 0) if err != nil { return err } @@ -985,37 +993,6 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return t.decNLeafs(wTx, 1) } -// upAfterDelete navigates back up the tree after a delete operation, updating -// the intermediate nodes and potentially removing nodes that are no longer needed. -func (t *Tree) upAfterDelete(wTx db.WriteTx, key []byte, siblings [][]byte, path []bool, currLvl int) ([]byte, error) { - if currLvl < 0 { - return key, nil - } - - var k, v []byte - var err error - if path[currLvl] { - k, v, err = t.newIntermediate(siblings[currLvl], key) - } else { - k, v, err = t.newIntermediate(key, siblings[currLvl]) - } - if err != nil { - return nil, err - } - - // If both children are empty, remove the intermediate node. - if bytes.Equal(siblings[currLvl], t.emptyHash) && bytes.Equal(key, t.emptyHash) { - return t.emptyHash, nil - } - - // Otherwise, store the updated intermediate node. - if err = wTx.Set(k, v); err != nil { - return nil, err - } - - return t.upAfterDelete(wTx, k, siblings, path, currLvl-1) -} - // decNLeafs decreases the number of leaves in the tree. func (t *Tree) decNLeafs(wTx db.WriteTx, n int) error { nLeafs, err := t.GetNLeafsWithTx(wTx) diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 89bc48556..89a2ef204 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -2,6 +2,7 @@ package arbo import ( "encoding/hex" + "fmt" "math" "math/big" "os" @@ -103,8 +104,7 @@ func testAdd(c *qt.C, hashFunc HashFunction, testVectors []string) { func TestAddBatch(t *testing.T) { c := qt.New(t) database := metadb.NewTest(t) - tree, err := NewTree(Config{Database: database, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) bLen := 32 @@ -120,8 +120,7 @@ func TestAddBatch(t *testing.T) { "296519252211642170490407814696803112091039265640052570497930797516015811235") database = metadb.NewTest(t) - tree2, err := NewTree(Config{Database: database, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree2, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) var keys, values [][]byte @@ -142,8 +141,7 @@ func TestAddBatch(t *testing.T) { func TestAddDifferentOrder(t *testing.T) { c := qt.New(t) database1 := metadb.NewTest(t) - tree1, err := NewTree(Config{Database: database1, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree1, err := NewTree(Config{Database: database1, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) bLen := 32 @@ -156,8 +154,7 @@ func TestAddDifferentOrder(t *testing.T) { } database2 := metadb.NewTest(t) - tree2, err := NewTree(Config{Database: database2, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree2, err := NewTree(Config{Database: database2, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) for i := 16 - 1; i >= 0; i-- { @@ -180,8 +177,7 @@ func TestAddDifferentOrder(t *testing.T) { func TestAddRepeatedIndex(t *testing.T) { c := qt.New(t) database := metadb.NewTest(t) - tree, err := NewTree(Config{Database: database, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) bLen := 32 @@ -197,8 +193,7 @@ func TestAddRepeatedIndex(t *testing.T) { func TestUpdate(t *testing.T) { c := qt.New(t) database := metadb.NewTest(t) - tree, err := NewTree(Config{Database: database, MaxLevels: 256, - HashFunction: HashFunctionPoseidon}) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionPoseidon}) c.Assert(err, qt.IsNil) bLen := 32 @@ -995,3 +990,125 @@ func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) { } } } + +func TestDiskSizeBench(t *testing.T) { + c := qt.New(t) + + nLeafs := 20_000 + printTestContext("TestDiskSizeBench: ", nLeafs, "Blake2b", "pebble") + + // prepare inputs + var ks, vs [][]byte + for i := 0; i < nLeafs; i++ { + k := randomBytes(32) + v := randomBytes(32) + ks = append(ks, k) + vs = append(vs, v) + } + + // create the database and tree + dbDir := t.TempDir() + database, err := metadb.New(db.TypePebble, dbDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + + tree, err := NewTree(Config{database, 256, DefaultThresholdNLeafs, HashFunctionBlake2b}) + c.Assert(err, qt.IsNil) + + countDBitems := func() int { + count := 0 + if err := tree.db.Iterate(nil, func(k, v []byte) bool { + count++ + return true + }); err != nil { + t.Fatal(err) + } + return count + } + + // add the leafs + start := time.Now() + tx := tree.db.WriteTx() + for i := 0; i < len(ks); i++ { + err = tree.AddWithTx(tx, ks[i], vs[i]) + c.Assert(err, qt.IsNil) + // fmt.Printf("add %x\n", ks[i]) + } + c.Assert(tx.Commit(), qt.IsNil) + printRes(" Add loop", time.Since(start)) + tree.dbg.print(" ") + size, err := dirSize(dbDir) + c.Assert(err, qt.IsNil) + printRes(" Disk size (add)", fmt.Sprintf("%d MiB", size/(1024*1024))) + printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + + // delete the leafs + start = time.Now() + tx = tree.db.WriteTx() + for i := 0; i < len(ks)/2; i++ { + err = tree.DeleteWithTx(tx, ks[i]) + c.Assert(err, qt.IsNil) + //fmt.Printf("deleted %x\n", ks[i]) + } + c.Assert(tx.Commit(), qt.IsNil) + printRes(" Delete loop", time.Since(start)) + tree.dbg.print(" ") + size, err = dirSize(dbDir) + c.Assert(err, qt.IsNil) + printRes(" Disk size (delete)", fmt.Sprintf("%d MiB", size/(1024*1024))) + printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + + // add the leafs deleted again + start = time.Now() + tx = tree.db.WriteTx() + for i := 0; i < len(ks)/2; i++ { + err = tree.AddWithTx(tx, ks[i], vs[i]) + c.Assert(err, qt.IsNil) + //fmt.Printf("added %x\n", ks[i]) + } + c.Assert(tx.Commit(), qt.IsNil) + printRes(" Add2 loop", time.Since(start)) + tree.dbg.print(" ") + size, err = dirSize(dbDir) + c.Assert(err, qt.IsNil) + printRes(" Disk size (add2)", fmt.Sprintf("%d MiB", size/(1024*1024))) + printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + + // update the leafs + start = time.Now() + tx = tree.db.WriteTx() + for i := 0; i < len(ks); i++ { + err = tree.UpdateWithTx(tx, ks[i], vs[len(ks)-i-1]) + c.Assert(err, qt.IsNil, qt.Commentf("k=%x", ks[i])) + //fmt.Printf("updated %x\n", ks[i]) + } + c.Assert(tx.Commit(), qt.IsNil) + printRes(" Update loop", time.Since(start)) + tree.dbg.print(" ") + size, err = dirSize(dbDir) + c.Assert(err, qt.IsNil) + printRes(" Disk size (update)", fmt.Sprintf("%d MiB", size/(1024*1024))) + printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + + start = time.Now() + c.Assert(tree.db.Compact(), qt.IsNil) + printRes(" Compact DB", time.Since(start)) + printRes(" Disk size (compact)", fmt.Sprintf("%d MiB", size/(1024*1024))) + printRes(" DB items", fmt.Sprintf("%d", countDBitems())) +} + +func dirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} From 443384b5d48769ff96b57e64ff802e88a7e83912 Mon Sep 17 00:00:00 2001 From: p4u Date: Tue, 29 Aug 2023 16:36:34 +0200 Subject: [PATCH 06/15] arbo test Signed-off-by: p4u --- tree/arbo/navigate.go | 168 ++++++++++++++++++++++++++++++++++ tree/arbo/tree.go | 201 +++++++---------------------------------- tree/arbo/tree_test.go | 10 +- 3 files changed, 210 insertions(+), 169 deletions(-) create mode 100644 tree/arbo/navigate.go diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go new file mode 100644 index 000000000..03de76551 --- /dev/null +++ b/tree/arbo/navigate.go @@ -0,0 +1,168 @@ +package arbo + +import ( + "bytes" + "encoding/hex" + "fmt" + + "go.vocdoni.io/dvote/db" +) + +// down goes down to the leaf recursively +func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, intermediates [][]byte, + path []bool, currLvl int, getLeaf bool) ([]byte, []byte, [][]byte, [][]byte, error) { + if currLvl > t.maxLevels { + return nil, nil, nil, nil, ErrMaxLevel + } + + var err error + var currValue []byte + if bytes.Equal(currKey, t.emptyHash) { + // empty value + return currKey, emptyValue, siblings, intermediates, nil + } + currValue, err = rTx.Get(currKey) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("could not get value for key %x: %w", currKey, err) + } + + switch currValue[0] { + case PrefixValueEmpty: // empty + fmt.Printf("newKey: %s, currKey: %s, currLvl: %d, currValue: %s\n", + hex.EncodeToString(newKey), hex.EncodeToString(currKey), + currLvl, hex.EncodeToString(currValue)) + panic("This point should not be reached, as the 'if currKey==t.emptyHash'" + + " above should avoid reaching this point. This panic is temporary" + + " for reporting purposes, will be deleted in future versions." + + " Please paste this log (including the previous log lines) in a" + + " new issue: https://go.vocdoni.io/dvote/tree/arbo/issues/new") // TMP + case PrefixValueLeaf: // leaf + if !bytes.Equal(currValue, emptyValue) { + if getLeaf { + return currKey, currValue, siblings, intermediates, nil + } + oldLeafKey, _ := ReadLeafValue(currValue) + if bytes.Equal(newKey, oldLeafKey) { + return nil, nil, nil, nil, ErrKeyAlreadyExists + } + + oldLeafKeyFull, err := keyPathFromKey(t.maxLevels, oldLeafKey) + if err != nil { + return nil, nil, nil, nil, err + } + + // if currKey is already used, go down until paths diverge + oldPath := getPath(t.maxLevels, oldLeafKeyFull) + siblings, err = t.downVirtually(siblings, currKey, newKey, oldPath, path, currLvl) + if err != nil { + return nil, nil, nil, nil, err + } + } + return currKey, currValue, siblings, intermediates, nil + case PrefixValueIntermediate: // intermediate + if len(currValue) != PrefixValueLen+t.hashFunction.Len()*2 { + return nil, nil, nil, nil, + fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)", + PrefixValueLen+t.hashFunction.Len()*2, len(currValue)) + } + intermediates[currLvl] = currKey + + // collect siblings while going down + if path[currLvl] { + // right + lChild, rChild := ReadIntermediateChilds(currValue) + siblings = append(siblings, lChild) + return t.down(rTx, newKey, rChild, siblings, intermediates, path, currLvl+1, getLeaf) + } + // left + lChild, rChild := ReadIntermediateChilds(currValue) + siblings = append(siblings, rChild) + return t.down(rTx, newKey, lChild, siblings, intermediates, path, currLvl+1, getLeaf) + default: + return nil, nil, nil, nil, ErrInvalidValuePrefix + } +} + +// up navigates back up the tree after a delete operation, updating +// the intermediate nodes and potentially removing nodes that are no longer needed. +func (t *Tree) up(wTx db.WriteTx, intermediates [][]byte, newKey []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) { + if currLvl < 0 { + return newKey, nil + } + + var k, v []byte + var err error + if path[currLvl+toLvl] { + k, v, err = t.newIntermediate(siblings[currLvl], newKey) + } else { + k, v, err = t.newIntermediate(newKey, siblings[currLvl]) + } + + if err != nil { + return nil, fmt.Errorf("could not compute intermediary node: %w", err) + } + + // If the new key is not the empty node, store it in the database + // extra empty childs are stored. find the way to remove them + if err = wTx.Set(k, v); err != nil { + return nil, err + } + + // If the node is modified, remove the old key + oldKey := intermediates[currLvl] + if !bytes.Equal(oldKey, k) && oldKey != nil { + fmt.Printf("Removing old key: %s\n", hex.EncodeToString(oldKey)) + if err := wTx.Delete(oldKey); err != nil { + return nil, fmt.Errorf("could not delete old key: %w", err) + } + } + + return t.up(wTx, intermediates, k, siblings, path, currLvl-1, toLvl) +} + +// newIntermediate takes the left & right keys of a intermediate node, and +// computes its hash. Returns the hash of the node, which is the node key, and a +// byte array that contains the value (which contains the left & right child +// keys) to store in the DB. +// [ 1 byte | 1 byte | N bytes | N bytes ] +// [ type of node | length of left key | left key | right key ] +func newIntermediate(hashFunc HashFunction, l, r []byte) ([]byte, []byte, error) { + b := make([]byte, PrefixValueLen+hashFunc.Len()*2) + b[0] = PrefixValueIntermediate + if len(l) > maxUint8 { + return nil, nil, fmt.Errorf("newIntermediate: len(l) > %v", maxUint8) + } + b[1] = byte(len(l)) + copy(b[PrefixValueLen:PrefixValueLen+hashFunc.Len()], l) + copy(b[PrefixValueLen+hashFunc.Len():], r) + + key, err := hashFunc.Hash(l, r) + if err != nil { + return nil, nil, err + } + + return key, b, nil +} + +// ReadIntermediateChilds reads from a byte array the two childs keys +func ReadIntermediateChilds(b []byte) ([]byte, []byte) { + if len(b) < PrefixValueLen { + return []byte{}, []byte{} + } + + lLen := b[1] + if len(b) < PrefixValueLen+int(lLen) { + return []byte{}, []byte{} + } + l := b[PrefixValueLen : PrefixValueLen+lLen] + r := b[PrefixValueLen+lLen:] + return l, r +} + +func getPath(numLevels int, k []byte) []bool { + path := make([]bool, numLevels) + for n := 0; n < numLevels; n++ { + path[n] = k[n/8]&(1<<(n%8)) != 0 + } + return path +} diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 206ebcc15..6947c374b 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -102,6 +102,9 @@ type Tree struct { // (check if it has been initialized) emptyHash []byte + // emptyNode is the hash of an empty node (with both childs empty) + emptyNode []byte + dbg *dbgStats } @@ -138,11 +141,18 @@ func NewTreeWithTx(wTx db.WriteTx, cfg Config) (*Tree, error) { if cfg.ThresholdNLeafs == 0 { cfg.ThresholdNLeafs = DefaultThresholdNLeafs } + t := Tree{db: cfg.Database, maxLevels: cfg.MaxLevels, thresholdNLeafs: cfg.ThresholdNLeafs, hashFunction: cfg.HashFunction} + t.emptyHash = make([]byte, t.hashFunction.Len()) // empty + var err error + t.emptyNode, _, err = t.newIntermediate(t.emptyHash, t.emptyHash) + if err != nil { + return nil, err + } - _, err := wTx.Get(dbKeyRoot) + _, err = wTx.Get(dbKeyRoot) if err == db.ErrKeyNotFound { // store new root 0 (empty) if err = wTx.Set(dbKeyRoot, t.emptyHash); err != nil { @@ -609,7 +619,9 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt // go down to the leaf var siblings [][]byte - _, _, siblings, err = t.down(wTx, k, root, siblings, path, fromLvl, false) + intermediates := make([][]byte, t.maxLevels) + + _, _, siblings, intermediates, err = t.down(wTx, k, root, siblings, intermediates, path, fromLvl, false) if err != nil { return nil, err } @@ -627,7 +639,7 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt // return the leafKey as root return leafKey, nil } - root, err = t.up(wTx, leafKey, siblings, path, len(siblings)-1, fromLvl) + root, err = t.up(wTx, intermediates, leafKey, siblings, path, len(siblings)-1, fromLvl) if err != nil { return nil, err } @@ -635,80 +647,6 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt return root, nil } -// down goes down to the leaf recursively -func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, - path []bool, currLvl int, getLeaf bool) ( - []byte, []byte, [][]byte, error) { - if currLvl > t.maxLevels { - return nil, nil, nil, ErrMaxLevel - } - - var err error - var currValue []byte - if bytes.Equal(currKey, t.emptyHash) { - // empty value - return currKey, emptyValue, siblings, nil - } - currValue, err = rTx.Get(currKey) - if err != nil { - return nil, nil, nil, err - } - - switch currValue[0] { - case PrefixValueEmpty: // empty - fmt.Printf("newKey: %s, currKey: %s, currLvl: %d, currValue: %s\n", - hex.EncodeToString(newKey), hex.EncodeToString(currKey), - currLvl, hex.EncodeToString(currValue)) - panic("This point should not be reached, as the 'if currKey==t.emptyHash'" + - " above should avoid reaching this point. This panic is temporary" + - " for reporting purposes, will be deleted in future versions." + - " Please paste this log (including the previous log lines) in a" + - " new issue: https://go.vocdoni.io/dvote/tree/arbo/issues/new") // TMP - case PrefixValueLeaf: // leaf - if !bytes.Equal(currValue, emptyValue) { - if getLeaf { - return currKey, currValue, siblings, nil - } - oldLeafKey, _ := ReadLeafValue(currValue) - if bytes.Equal(newKey, oldLeafKey) { - return nil, nil, nil, ErrKeyAlreadyExists - } - - oldLeafKeyFull, err := keyPathFromKey(t.maxLevels, oldLeafKey) - if err != nil { - return nil, nil, nil, err - } - - // if currKey is already used, go down until paths diverge - oldPath := getPath(t.maxLevels, oldLeafKeyFull) - siblings, err = t.downVirtually(siblings, currKey, newKey, oldPath, path, currLvl) - if err != nil { - return nil, nil, nil, err - } - } - return currKey, currValue, siblings, nil - case PrefixValueIntermediate: // intermediate - if len(currValue) != PrefixValueLen+t.hashFunction.Len()*2 { - return nil, nil, nil, - fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)", - PrefixValueLen+t.hashFunction.Len()*2, len(currValue)) - } - // collect siblings while going down - if path[currLvl] { - // right - lChild, rChild := ReadIntermediateChilds(currValue) - siblings = append(siblings, lChild) - return t.down(rTx, newKey, rChild, siblings, path, currLvl+1, getLeaf) - } - // left - lChild, rChild := ReadIntermediateChilds(currValue) - siblings = append(siblings, rChild) - return t.down(rTx, newKey, lChild, siblings, path, currLvl+1, getLeaf) - default: - return nil, nil, nil, ErrInvalidValuePrefix - } -} - // downVirtually is used when in a leaf already exists, and a new leaf which // shares the path until the existing leaf is being added func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath, @@ -733,43 +671,6 @@ func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath, return siblings, nil } -// up navigates back up the tree after a delete operation, updating -// the intermediate nodes and potentially removing nodes that are no longer needed. -func (t *Tree) up(wTx db.WriteTx, key []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) { - if currLvl < 0 { - return key, nil - } - - var k, v []byte - var err error - if path[currLvl+toLvl] { - k, v, err = t.newIntermediate(siblings[currLvl], key) - } else { - k, v, err = t.newIntermediate(key, siblings[currLvl]) - } - if err != nil { - return nil, err - } - - // If both children are empty, remove the intermediate node and its children. - if bytes.Equal(k, t.emptyHash) && bytes.Equal(v, t.emptyHash) { - if err := wTx.Delete(key); err != nil { - return nil, err - } - if err := wTx.Delete(siblings[currLvl]); err != nil { - return nil, err - } - return t.emptyHash, wTx.Set(k, t.emptyHash) - } - - // Otherwise, store the updated intermediate node. - if err = wTx.Set(k, v); err != nil { - return nil, err - } - - return t.up(wTx, k, siblings, path, currLvl-1, toLvl) -} - func (t *Tree) newLeafValue(k, v []byte) ([]byte, []byte, error) { t.dbg.incHash() return newLeafValue(t.hashFunction, k, v) @@ -817,53 +718,6 @@ func (t *Tree) newIntermediate(l, r []byte) ([]byte, []byte, error) { return newIntermediate(t.hashFunction, l, r) } -// newIntermediate takes the left & right keys of a intermediate node, and -// computes its hash. Returns the hash of the node, which is the node key, and a -// byte array that contains the value (which contains the left & right child -// keys) to store in the DB. -// [ 1 byte | 1 byte | N bytes | N bytes ] -// [ type of node | length of left key | left key | right key ] -func newIntermediate(hashFunc HashFunction, l, r []byte) ([]byte, []byte, error) { - b := make([]byte, PrefixValueLen+hashFunc.Len()*2) - b[0] = PrefixValueIntermediate - if len(l) > maxUint8 { - return nil, nil, fmt.Errorf("newIntermediate: len(l) > %v", maxUint8) - } - b[1] = byte(len(l)) - copy(b[PrefixValueLen:PrefixValueLen+hashFunc.Len()], l) - copy(b[PrefixValueLen+hashFunc.Len():], r) - - key, err := hashFunc.Hash(l, r) - if err != nil { - return nil, nil, err - } - - return key, b, nil -} - -// ReadIntermediateChilds reads from a byte array the two childs keys -func ReadIntermediateChilds(b []byte) ([]byte, []byte) { - if len(b) < PrefixValueLen { - return []byte{}, []byte{} - } - - lLen := b[1] - if len(b) < PrefixValueLen+int(lLen) { - return []byte{}, []byte{} - } - l := b[PrefixValueLen : PrefixValueLen+lLen] - r := b[PrefixValueLen+lLen:] - return l, r -} - -func getPath(numLevels int, k []byte) []bool { - path := make([]bool, numLevels) - for n := 0; n < numLevels; n++ { - path[n] = k[n/8]&(1<<(n%8)) != 0 - } - return path -} - // Update updates the value for a given existing key. If the given key does not // exist, returns an error. func (t *Tree) Update(k, v []byte) error { @@ -899,7 +753,9 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { } var siblings [][]byte - _, valueAtBottom, siblings, err := t.down(wTx, k, root, siblings, path, 0, true) + intermediates := make([][]byte, t.maxLevels) + + _, valueAtBottom, siblings, intermediates, err := t.down(wTx, k, root, siblings, intermediates, path, 0, true) if err != nil { return err } @@ -921,7 +777,7 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { if len(siblings) == 0 { return t.setRoot(wTx, leafKey) } - root, err = t.up(wTx, leafKey, siblings, path, len(siblings)-1, 0) + root, err = t.up(wTx, intermediates, leafKey, siblings, path, len(siblings)-1, 0) if err != nil { return err } @@ -964,8 +820,10 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { // Navigate down the tree to find the key and collect siblings. var siblings [][]byte + intermediates := make([][]byte, t.maxLevels) + path := getPath(t.maxLevels, k) - _, _, siblings, err = t.down(wTx, k, root, siblings, path, 0, true) + leafKey, _, siblings, intermediates, err := t.down(wTx, k, root, siblings, intermediates, path, 0, true) if err != nil { if err == ErrKeyNotFound { // Key not found, nothing to delete. @@ -974,12 +832,17 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return err } + // delete the leaf key + if err := wTx.Delete(leafKey); err != nil { + return fmt.Errorf("error deleting key %x: %w", leafKey, err) + } + // Navigate back up the tree, updating the intermediate nodes. if len(siblings) == 0 { // The tree is empty after deletion. return t.setRoot(wTx, t.emptyHash) } - newRoot, err := t.up(wTx, t.emptyHash, siblings, path, len(siblings)-1, 0) + newRoot, err := t.up(wTx, intermediates, t.emptyHash, siblings, path, len(siblings)-1, 0) if err != nil { return err } @@ -1025,7 +888,9 @@ func (t *Tree) GenProofWithTx(rTx db.Reader, k []byte) ([]byte, []byte, []byte, // go down to the leaf var siblings [][]byte - _, value, siblings, err := t.down(rTx, k, root, siblings, path, 0, true) + intermediates := make([][]byte, t.maxLevels) + + _, value, siblings, intermediates, err := t.down(rTx, k, root, siblings, intermediates, path, 0, true) if err != nil { return nil, nil, nil, false, err } @@ -1174,7 +1039,9 @@ func (t *Tree) GetWithTx(rTx db.Reader, k []byte) ([]byte, []byte, error) { // go down to the leaf var siblings [][]byte - _, value, _, err := t.down(rTx, k, root, siblings, path, 0, true) + intermediates := make([][]byte, t.maxLevels) + + _, value, _, intermediates, err := t.down(rTx, k, root, siblings, intermediates, path, 0, true) if err != nil { return nil, nil, err } diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 89a2ef204..7bfe37295 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -994,7 +994,7 @@ func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) { func TestDiskSizeBench(t *testing.T) { c := qt.New(t) - nLeafs := 20_000 + nLeafs := 20 printTestContext("TestDiskSizeBench: ", nLeafs, "Blake2b", "pebble") // prepare inputs @@ -1020,6 +1020,7 @@ func TestDiskSizeBench(t *testing.T) { countDBitems := func() int { count := 0 if err := tree.db.Iterate(nil, func(k, v []byte) bool { + //fmt.Printf("db item: %x\n", k) count++ return true }); err != nil { @@ -1027,7 +1028,6 @@ func TestDiskSizeBench(t *testing.T) { } return count } - // add the leafs start := time.Now() tx := tree.db.WriteTx() @@ -1044,6 +1044,8 @@ func TestDiskSizeBench(t *testing.T) { printRes(" Disk size (add)", fmt.Sprintf("%d MiB", size/(1024*1024))) printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + tree.PrintGraphviz(nil) + // delete the leafs start = time.Now() tx = tree.db.WriteTx() @@ -1060,6 +1062,10 @@ func TestDiskSizeBench(t *testing.T) { printRes(" Disk size (delete)", fmt.Sprintf("%d MiB", size/(1024*1024))) printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + tree.PrintGraphviz(nil) + + return + // add the leafs deleted again start = time.Now() tx = tree.db.WriteTx() From 6d45964008d68009945057962cbfe7432aa31ad5 Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 30 Aug 2023 00:52:02 +0200 Subject: [PATCH 07/15] metadb: add goleveldb implementation Having a second implementation of the db interface is useful for testing purposes. We can easily switch from pebble to leveldb in order to test something in a different storage layer. In addition add a Compact() method to the db interface. --- db/goleveldb/goleveldb.go | 156 +++++++++++++++++++++++++++++++++ db/goleveldb/goleveldb_test.go | 52 +++++++++++ db/interface.go | 8 +- db/metadb/metadb.go | 6 ++ db/pebbledb/pebledb.go | 25 +++++- db/prefixeddb/prefixeddb.go | 5 ++ 6 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 db/goleveldb/goleveldb.go create mode 100644 db/goleveldb/goleveldb_test.go diff --git a/db/goleveldb/goleveldb.go b/db/goleveldb/goleveldb.go new file mode 100644 index 000000000..730462ce6 --- /dev/null +++ b/db/goleveldb/goleveldb.go @@ -0,0 +1,156 @@ +package goleveldb + +import ( + "bytes" + "errors" + "fmt" + "sync" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" + "go.vocdoni.io/dvote/db" +) + +type LevelDB struct { + db *leveldb.DB +} + +// Ensure that LevelDB implements the db.Database interface +var _ db.Database = (*LevelDB)(nil) + +// New returns a LevelDB which implements the db.Database interface +func New(opts db.Options) (*LevelDB, error) { + // Open the LevelDB database + db, err := leveldb.OpenFile(opts.Path, &opt.Options{}) + if err != nil { + return nil, fmt.Errorf("could not open leveldb: %w", err) + } + return &LevelDB{ + db: db, + }, nil +} + +func (d *LevelDB) Close() error { + return d.db.Close() +} + +func (d *LevelDB) WriteTx() db.WriteTx { + return &WriteTx{ + db: d.db, + batch: new(leveldb.Batch), + } +} + +func (d *LevelDB) Get(key []byte) ([]byte, error) { + val, err := d.db.Get(key, nil) + if errors.Is(err, leveldb.ErrNotFound) { + return nil, db.ErrKeyNotFound + } + if err != nil { + return nil, err + } + return val, nil +} + +func (d *LevelDB) Iterate(prefix []byte, callback func(key, value []byte) bool) error { + iter := d.db.NewIterator(util.BytesPrefix(prefix), nil) + defer iter.Release() + for iter.Next() { + if !callback(iter.Key(), iter.Value()) { + break + } + } + return iter.Error() +} + +func (d *LevelDB) Set(key, value []byte) error { + return d.db.Put(key, value, nil) +} + +func (d *LevelDB) Delete(key []byte) error { + return d.db.Delete(key, nil) +} + +func (d *LevelDB) Commit(batch *leveldb.Batch) error { + return d.db.Write(batch, nil) +} + +// Compact implements the db.Database.Compact interface method. +func (d *LevelDB) Compact() error { + return d.db.CompactRange(util.Range{}) +} + +// WriteTx implements the interface db.WriteTx for goleveldb +type WriteTx struct { + batch *leveldb.Batch + db *leveldb.DB + inMemBatch sync.Map +} + +// check that WriteTx implements the db.WriteTx interface +var _ db.WriteTx = (*WriteTx)(nil) + +func (tx *WriteTx) Get(k []byte) ([]byte, error) { + val, ok := tx.inMemBatch.Load(string(k)) + if !ok { + val, err := tx.db.Get(k, nil) + if errors.Is(err, leveldb.ErrNotFound) { + return nil, db.ErrKeyNotFound + } + return val, err + } + return val.([]byte), nil +} + +func (tx *WriteTx) Iterate(prefix []byte, callback func(k, v []byte) bool) error { + inMemory := make(map[string]bool) + tx.inMemBatch.Range(func(k, v any) bool { + keyBytes := []byte(k.(string)) + if bytes.HasPrefix(keyBytes, prefix) { + inMemory[string(keyBytes)] = true + return callback(keyBytes, v.([]byte)) + } + return true + }) + iter := tx.db.NewIterator(util.BytesPrefix(prefix), nil) + defer iter.Release() + for iter.Next() { + if inMemory[string(iter.Key())] { + continue + } + if !callback(iter.Key(), iter.Value()) { + break + } + } + return iter.Error() +} + +func (tx *WriteTx) Set(k, v []byte) error { + tx.batch.Put(k, v) + tx.inMemBatch.Store(string(k), v) + return nil +} + +func (tx *WriteTx) Delete(k []byte) error { + tx.batch.Delete(k) + tx.inMemBatch.Delete(string(k)) + return nil +} + +func (tx *WriteTx) Apply(otherTx db.WriteTx) error { + return otherTx.Iterate(nil, func(k, v []byte) bool { + tx.inMemBatch.Store(string(k), v) + tx.batch.Put(k, v) + return true + }) +} + +func (tx *WriteTx) Commit() error { + return tx.db.Write(tx.batch, nil) +} + +func (tx *WriteTx) Discard() { + tx.batch.Reset() + tx.inMemBatch = sync.Map{} +} diff --git a/db/goleveldb/goleveldb_test.go b/db/goleveldb/goleveldb_test.go new file mode 100644 index 000000000..b6d2d4240 --- /dev/null +++ b/db/goleveldb/goleveldb_test.go @@ -0,0 +1,52 @@ +package goleveldb + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/internal/dbtest" + "go.vocdoni.io/dvote/db/prefixeddb" +) + +func TestWriteTx(t *testing.T) { + database, err := New(db.Options{Path: t.TempDir()}) + qt.Assert(t, err, qt.IsNil) + + dbtest.TestWriteTx(t, database) +} + +func TestIterate(t *testing.T) { + database, err := New(db.Options{Path: t.TempDir()}) + qt.Assert(t, err, qt.IsNil) + + dbtest.TestIterate(t, database) +} + +func TestWriteTxApply(t *testing.T) { + database, err := New(db.Options{Path: t.TempDir()}) + qt.Assert(t, err, qt.IsNil) + + dbtest.TestWriteTxApply(t, database) +} + +func TestWriteTxApplyPrefixed(t *testing.T) { + database, err := New(db.Options{Path: t.TempDir()}) + qt.Assert(t, err, qt.IsNil) + + prefix := []byte("one") + dbWithPrefix := prefixeddb.NewPrefixedDatabase(database, prefix) + + dbtest.TestWriteTxApplyPrefixed(t, database, dbWithPrefix) +} + +// NOTE: This test fails. pebble.Batch doesn't detect conflicts. Moreover, +// reads from a pebble.Batch return the last version from the Database, even if +// the update was made after the pebble.Batch was created. Basically it's not +// a Transaction, but a Batch of write operations. +// func TestConcurrentWriteTx(t *testing.T) { +// database, err := New(db.Options{Path: t.TempDir()}) +// qt.Assert(t, err, qt.IsNil) +// +// dbtest.TestConcurrentWriteTx(t, database) +// } diff --git a/db/interface.go b/db/interface.go index 11af7d28c..7aa1c37c6 100644 --- a/db/interface.go +++ b/db/interface.go @@ -6,7 +6,10 @@ import ( ) // TypePebble defines the type of db that uses PebbleDB -const TypePebble = "pebble" +const ( + TypePebble = "pebble" + TypeLevelDB = "leveldb" +) // ErrKeyNotFound is used to indicate that a key does not exist in the db. var ErrKeyNotFound = fmt.Errorf("key not found") @@ -33,6 +36,9 @@ type Database interface { // WriteTx creates a new write transaction. WriteTx() WriteTx + + // Compact compacts the underlying storage. + Compact() error } // Reader contains the read-only database operations. diff --git a/db/metadb/metadb.go b/db/metadb/metadb.go index ded65ca96..09f4afb71 100644 --- a/db/metadb/metadb.go +++ b/db/metadb/metadb.go @@ -6,6 +6,7 @@ import ( "testing" "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/goleveldb" "go.vocdoni.io/dvote/db/pebbledb" ) @@ -19,6 +20,11 @@ func New(typ, dir string) (db.Database, error) { if err != nil { return nil, err } + case db.TypeLevelDB: + database, err = goleveldb.New(opts) + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("invalid dbType: %q. Available types: %q", typ, db.TypePebble) } diff --git a/db/pebbledb/pebledb.go b/db/pebbledb/pebledb.go index aa359664b..685f921b1 100644 --- a/db/pebbledb/pebledb.go +++ b/db/pebbledb/pebledb.go @@ -110,7 +110,13 @@ func New(opts db.Options) (*PebbleDB, error) { if err := os.MkdirAll(opts.Path, os.ModePerm); err != nil { return nil, err } - o := &pebble.Options{} + o := &pebble.Options{ + Levels: []pebble.LevelOptions{ + { + Compression: pebble.SnappyCompression, + }, + }, + } db, err := pebble.Open(opts.Path, o) if err != nil { return nil, err @@ -155,3 +161,20 @@ func keyUpperBound(b []byte) []byte { func (db *PebbleDB) Iterate(prefix []byte, callback func(k, v []byte) bool) (err error) { return iterate(db.db, prefix, callback) } + +// Compact implements the db.Database.Compact interface method +func (db *PebbleDB) Compact() error { + // from https://github.com/cockroachdb/pebble/issues/1474#issuecomment-1022313365 + iter := db.db.NewIter(nil) + var first, last []byte + if iter.First() { + first = append(first, iter.Key()...) + } + if iter.Last() { + last = append(last, iter.Key()...) + } + if err := iter.Close(); err != nil { + return err + } + return db.db.Compact(first, last, true) +} diff --git a/db/prefixeddb/prefixeddb.go b/db/prefixeddb/prefixeddb.go index c51919537..c5998c9ef 100644 --- a/db/prefixeddb/prefixeddb.go +++ b/db/prefixeddb/prefixeddb.go @@ -41,6 +41,11 @@ func (d *PrefixedDatabase) Close() error { return d.db.Close() } +// Compact implements the db.Database.Compact interface method. +func (d *PrefixedDatabase) Compact() error { + return d.db.Compact() +} + // WriteTx returns a db.WriteTx func (d *PrefixedDatabase) WriteTx() db.WriteTx { return NewPrefixedWriteTx(d.db.WriteTx(), d.prefix) From d0d27edf53e3f11256db0d6e0244ecf9c4bf6f66 Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 30 Aug 2023 00:57:10 +0200 Subject: [PATCH 08/15] arbo: fix the clean up of nodes after delete/update/add Remove all intermediate nodes from the database once they are not necessary anymore. Break up the code into smaller Go files. --- tree/arbo/batch.go | 325 +++++++++++++++++++ tree/arbo/navigate.go | 84 +++-- tree/arbo/proof.go | 193 ++++++++++++ tree/arbo/tree.go | 686 ++++------------------------------------- tree/arbo/tree_test.go | 8 +- tree/arbo/utils.go | 86 ++++++ 6 files changed, 714 insertions(+), 668 deletions(-) create mode 100644 tree/arbo/batch.go create mode 100644 tree/arbo/proof.go diff --git a/tree/arbo/batch.go b/tree/arbo/batch.go new file mode 100644 index 000000000..5d84c1d58 --- /dev/null +++ b/tree/arbo/batch.go @@ -0,0 +1,325 @@ +package arbo + +import ( + "bytes" + "fmt" + "math" + "runtime" + "sync" + + "go.vocdoni.io/dvote/db" +) + +// AddBatch adds a batch of key-values to the Tree. Returns an array containing +// the indexes of the keys failed to add. Supports empty values as input +// parameters, which is equivalent to 0 valued byte array. +func (t *Tree) AddBatch(keys, values [][]byte) ([]Invalid, error) { + wTx := t.db.WriteTx() + defer wTx.Discard() + + invalids, err := t.AddBatchWithTx(wTx, keys, values) + if err != nil { + return invalids, err + } + return invalids, wTx.Commit() +} + +// AddBatchWithTx does the same than the AddBatch method, but allowing to pass +// the db.WriteTx that is used. The db.WriteTx will not be committed inside +// this method. +func (t *Tree) AddBatchWithTx(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { + t.Lock() + defer t.Unlock() + + if !t.editable() { + return nil, ErrSnapshotNotEditable + } + + e := []byte{} + // equal the number of keys & values + if len(keys) > len(values) { + // add missing values + for i := len(values); i < len(keys); i++ { + values = append(values, e) + } + } else if len(keys) < len(values) { + // crop extra values + values = values[:len(keys)] + } + + nLeafs, err := t.GetNLeafsWithTx(wTx) + if err != nil { + return nil, err + } + if nLeafs > t.thresholdNLeafs { + return t.addBatchInDisk(wTx, keys, values) + } + return t.addBatchInMemory(wTx, keys, values) +} + +func (t *Tree) addBatchInDisk(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { + nCPU := flp2(runtime.NumCPU()) + if nCPU == 1 || len(keys) < nCPU { + var invalids []Invalid + for i := 0; i < len(keys); i++ { + if err := t.addWithTx(wTx, keys[i], values[i]); err != nil { + invalids = append(invalids, Invalid{i, err}) + } + } + return invalids, nil + } + + kvs, invalids, err := keysValuesToKvs(t.maxLevels, keys, values) + if err != nil { + return nil, err + } + + buckets := splitInBuckets(kvs, nCPU) + + root, err := t.RootWithTx(wTx) + if err != nil { + return nil, err + } + + l := int(math.Log2(float64(nCPU))) + subRoots, err := t.getSubRootsAtLevel(wTx, root, l+1) + if err != nil { + return nil, err + } + if len(subRoots) != nCPU { + // Already populated Tree but Unbalanced. + + // add one key at each bucket, and then continue with the flow + for i := 0; i < len(buckets); i++ { + // add one leaf of the bucket, if there is an error when + // adding the k-v, try to add the next one of the bucket + // (until one is added) + inserted := -1 + for j := 0; j < len(buckets[i]); j++ { + if newRoot, err := t.add(wTx, root, 0, + buckets[i][j].k, buckets[i][j].v); err == nil { + inserted = j + root = newRoot + break + } + } + + // remove the inserted element from buckets[i] + if inserted != -1 { + buckets[i] = append(buckets[i][:inserted], buckets[i][inserted+1:]...) + } + } + subRoots, err = t.getSubRootsAtLevel(wTx, root, l+1) + if err != nil { + return nil, err + } + } + + if len(subRoots) != nCPU { + return nil, fmt.Errorf("this error should not be reached."+ + " len(subRoots) != nCPU, len(subRoots)=%d, nCPU=%d."+ + " Please report it in a new issue:"+ + " https://go.vocdoni.io/dvote/tree/arbo/issues/new", len(subRoots), nCPU) + } + + invalidsInBucket := make([][]Invalid, nCPU) + txs := make([]db.WriteTx, nCPU) + for i := 0; i < nCPU; i++ { + txs[i] = t.db.WriteTx() + err := txs[i].Apply(wTx) + if err != nil { + return nil, err + } + } + + var wg sync.WaitGroup + wg.Add(nCPU) + for i := 0; i < nCPU; i++ { + go func(cpu int) { + // use different wTx for each cpu, after once all + // are done, iter over the cpuWTxs and copy their + // content into the main wTx + for j := 0; j < len(buckets[cpu]); j++ { + newSubRoot, err := t.add(txs[cpu], subRoots[cpu], + l, buckets[cpu][j].k, buckets[cpu][j].v) + if err != nil { + invalidsInBucket[cpu] = append(invalidsInBucket[cpu], + Invalid{buckets[cpu][j].pos, err}) + continue + } + // if there has not been errors, set the new subRoots[cpu] + subRoots[cpu] = newSubRoot + } + wg.Done() + }(i) + } + wg.Wait() + + for i := 0; i < nCPU; i++ { + if err := wTx.Apply(txs[i]); err != nil { + return nil, err + } + txs[i].Discard() + } + + for i := 0; i < len(invalidsInBucket); i++ { + invalids = append(invalids, invalidsInBucket[i]...) + } + + newRoot, err := t.upFromSubRoots(wTx, subRoots) + if err != nil { + return nil, err + } + + // update dbKeyNLeafs + if err := t.SetRootWithTx(wTx, newRoot); err != nil { + return nil, err + } + + // update nLeafs + if err := t.incNLeafs(wTx, len(keys)-len(invalids)); err != nil { + return nil, err + } + + return invalids, nil +} + +func (t *Tree) upFromSubRoots(wTx db.WriteTx, subRoots [][]byte) ([]byte, error) { + // is a method of Tree just to get access to t.hashFunction and + // t.emptyHash. + + // go up from subRoots to up, storing nodes in the given WriteTx + // once up at the root, store it in the WriteTx using the dbKeyRoot + if len(subRoots) == 1 { + return subRoots[0], nil + } + // get the subRoots values to know the node types of each subRoot + nodeTypes := make([]byte, len(subRoots)) + for i := 0; i < len(subRoots); i++ { + if bytes.Equal(subRoots[i], t.emptyHash) { + nodeTypes[i] = PrefixValueEmpty + continue + } + v, err := wTx.Get(subRoots[i]) + if err != nil { + return nil, err + } + nodeTypes[i] = v[0] + } + + var newSubRoots [][]byte + for i := 0; i < len(subRoots); i += 2 { + if (bytes.Equal(subRoots[i], t.emptyHash) && bytes.Equal(subRoots[i+1], t.emptyHash)) || + (nodeTypes[i] == PrefixValueLeaf && bytes.Equal(subRoots[i+1], t.emptyHash)) { + // when both sub nodes are empty, the parent is also empty + // or + // when 1st sub node is a leaf but the 2nd is empty, the + // leaf is used as 'parent' + + newSubRoots = append(newSubRoots, subRoots[i]) + continue + } + if bytes.Equal(subRoots[i], t.emptyHash) && nodeTypes[i+1] == PrefixValueLeaf { + // when 2nd sub node is a leaf but the 1st is empty, + // the leaf is used as 'parent' + newSubRoots = append(newSubRoots, subRoots[i+1]) + continue + } + + k, v, err := t.newIntermediate(subRoots[i], subRoots[i+1]) + if err != nil { + return nil, err + } + // store k-v to db + if err = wTx.Set(k, v); err != nil { + return nil, err + } + newSubRoots = append(newSubRoots, k) + } + + return t.upFromSubRoots(wTx, newSubRoots) +} + +func (t *Tree) getSubRootsAtLevel(rTx db.Reader, root []byte, l int) ([][]byte, error) { + // go at level l and return each node key, where each node key is the + // subRoot of the subTree that starts there + + var subRoots [][]byte + err := t.iterWithStop(rTx, root, 0, func(currLvl int, k, v []byte) bool { + if currLvl == l && !bytes.Equal(k, t.emptyHash) { + subRoots = append(subRoots, k) + } + if currLvl >= l { + return true // to stop the iter from going down + } + return false + }) + + return subRoots, err +} + +func (t *Tree) addBatchInMemory(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { + vt, err := t.loadVT(wTx) + if err != nil { + return nil, err + } + + invalids, err := vt.addBatch(keys, values) + if err != nil { + return nil, err + } + + // once the VirtualTree is build, compute the hashes + pairs, err := vt.computeHashes() + if err != nil { + // currently invalids in computeHashes are not counted, + // but should not be needed, as if there is an error there is + // nothing stored in the db and the error is returned + return nil, err + } + + // store pairs in db + for i := 0; i < len(pairs); i++ { + if err := wTx.Set(pairs[i][0], pairs[i][1]); err != nil { + return nil, err + } + } + + // store root (from the vt) to db + if vt.root != nil { + if err := wTx.Set(dbKeyRoot, vt.root.h); err != nil { + return nil, err + } + } + + // update nLeafs + if err := t.incNLeafs(wTx, len(keys)-len(invalids)); err != nil { + return nil, err + } + + return invalids, nil +} + +// loadVT loads a new virtual tree (vt) from the current Tree, which contains +// the same leafs. +func (t *Tree) loadVT(rTx db.Reader) (vt, error) { + vt := newVT(t.maxLevels, t.hashFunction) + vt.params.dbg = t.dbg + var callbackErr error + err := t.IterateWithStopWithTx(rTx, nil, func(_ int, k, v []byte) bool { + if v[0] != PrefixValueLeaf { + return false + } + leafK, leafV := ReadLeafValue(v) + if err := vt.add(0, leafK, leafV); err != nil { + callbackErr = err + return true + } + return false + }) + if callbackErr != nil { + return vt, callbackErr + } + + return vt, err +} diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go index 03de76551..16940f12d 100644 --- a/tree/arbo/navigate.go +++ b/tree/arbo/navigate.go @@ -9,21 +9,21 @@ import ( ) // down goes down to the leaf recursively -func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, intermediates [][]byte, - path []bool, currLvl int, getLeaf bool) ([]byte, []byte, [][]byte, [][]byte, error) { +func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, intermediates *[][]byte, + path []bool, currLvl int, getLeaf bool) ([]byte, []byte, [][]byte, error) { if currLvl > t.maxLevels { - return nil, nil, nil, nil, ErrMaxLevel + return nil, nil, nil, ErrMaxLevel } var err error var currValue []byte if bytes.Equal(currKey, t.emptyHash) { // empty value - return currKey, emptyValue, siblings, intermediates, nil + return currKey, emptyValue, siblings, nil } currValue, err = rTx.Get(currKey) if err != nil { - return nil, nil, nil, nil, fmt.Errorf("could not get value for key %x: %w", currKey, err) + return nil, nil, nil, fmt.Errorf("could not get value for key %x: %w", currKey, err) } switch currValue[0] { @@ -39,33 +39,33 @@ func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, in case PrefixValueLeaf: // leaf if !bytes.Equal(currValue, emptyValue) { if getLeaf { - return currKey, currValue, siblings, intermediates, nil + return currKey, currValue, siblings, nil } oldLeafKey, _ := ReadLeafValue(currValue) if bytes.Equal(newKey, oldLeafKey) { - return nil, nil, nil, nil, ErrKeyAlreadyExists + return nil, nil, nil, ErrKeyAlreadyExists } oldLeafKeyFull, err := keyPathFromKey(t.maxLevels, oldLeafKey) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } // if currKey is already used, go down until paths diverge oldPath := getPath(t.maxLevels, oldLeafKeyFull) siblings, err = t.downVirtually(siblings, currKey, newKey, oldPath, path, currLvl) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } } - return currKey, currValue, siblings, intermediates, nil + return currKey, currValue, siblings, nil case PrefixValueIntermediate: // intermediate if len(currValue) != PrefixValueLen+t.hashFunction.Len()*2 { - return nil, nil, nil, nil, + return nil, nil, nil, fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)", PrefixValueLen+t.hashFunction.Len()*2, len(currValue)) } - intermediates[currLvl] = currKey + *intermediates = append(*intermediates, currKey) // collect siblings while going down if path[currLvl] { @@ -79,17 +79,13 @@ func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, in siblings = append(siblings, rChild) return t.down(rTx, newKey, lChild, siblings, intermediates, path, currLvl+1, getLeaf) default: - return nil, nil, nil, nil, ErrInvalidValuePrefix + return nil, nil, nil, ErrInvalidValuePrefix } } -// up navigates back up the tree after a delete operation, updating -// the intermediate nodes and potentially removing nodes that are no longer needed. -func (t *Tree) up(wTx db.WriteTx, intermediates [][]byte, newKey []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) { - if currLvl < 0 { - return newKey, nil - } - +// up navigates back up the tree, updating the intermediate nodes and potentially +// removing nodes that are no longer needed. +func (t *Tree) up(wTx db.WriteTx, newKey []byte, siblings [][]byte, path []bool, currLvl, toLvl int) ([]byte, error) { var k, v []byte var err error if path[currLvl+toLvl] { @@ -97,27 +93,51 @@ func (t *Tree) up(wTx db.WriteTx, intermediates [][]byte, newKey []byte, sibling } else { k, v, err = t.newIntermediate(newKey, siblings[currLvl]) } - if err != nil { return nil, fmt.Errorf("could not compute intermediary node: %w", err) } - // If the new key is not the empty node, store it in the database - // extra empty childs are stored. find the way to remove them - if err = wTx.Set(k, v); err != nil { - return nil, err + // store the new intermediate node + if bytes.Equal(k, t.emptyNode) { + // if both children are empty, the parent is empty too, we return the empty hash, + // so next up() calls know it + k = t.emptyHash + } else { + // if the parent is not empty, store it + if err = wTx.Set(k, v); err != nil { + return nil, err + } + } + + if currLvl == 0 { + // reached the root + return k, nil } - // If the node is modified, remove the old key - oldKey := intermediates[currLvl] - if !bytes.Equal(oldKey, k) && oldKey != nil { - fmt.Printf("Removing old key: %s\n", hex.EncodeToString(oldKey)) - if err := wTx.Delete(oldKey); err != nil { - return nil, fmt.Errorf("could not delete old key: %w", err) + return t.up(wTx, k, siblings, path, currLvl-1, toLvl) +} + +// downVirtually is used when in a leaf already exists, and a new leaf which +// shares the path until the existing leaf is being added +func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath, newPath []bool, currLvl int) ([][]byte, error) { + var err error + if currLvl > t.maxLevels-1 { + return nil, ErrMaxVirtualLevel + } + + if oldPath[currLvl] == newPath[currLvl] { + siblings = append(siblings, t.emptyHash) + + siblings, err = t.downVirtually(siblings, oldKey, newKey, oldPath, newPath, currLvl+1) + if err != nil { + return nil, err } + return siblings, nil } + // reached the divergence + siblings = append(siblings, oldKey) - return t.up(wTx, intermediates, k, siblings, path, currLvl-1, toLvl) + return siblings, nil } // newIntermediate takes the left & right keys of a intermediate node, and diff --git a/tree/arbo/proof.go b/tree/arbo/proof.go new file mode 100644 index 000000000..dbf54bf06 --- /dev/null +++ b/tree/arbo/proof.go @@ -0,0 +1,193 @@ +package arbo + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + + "go.vocdoni.io/dvote/db" +) + +// GenProof generates a MerkleTree proof for the given key. The leaf value is +// returned, together with the packed siblings of the proof, and a boolean +// parameter that indicates if the proof is of existence (true) or not (false). +func (t *Tree) GenProof(k []byte) ([]byte, []byte, []byte, bool, error) { + return t.GenProofWithTx(t.db, k) +} + +// GenProofWithTx does the same than the GenProof method, but allowing to pass +// the db.ReadTx that is used. +func (t *Tree) GenProofWithTx(rTx db.Reader, k []byte) ([]byte, []byte, []byte, bool, error) { + keyPath, err := keyPathFromKey(t.maxLevels, k) + if err != nil { + return nil, nil, nil, false, err + } + path := getPath(t.maxLevels, keyPath) + + root, err := t.RootWithTx(rTx) + if err != nil { + return nil, nil, nil, false, err + } + + // go down to the leaf + var siblings, intermediates [][]byte + + _, value, siblings, err := t.down(rTx, k, root, siblings, &intermediates, path, 0, true) + if err != nil { + return nil, nil, nil, false, err + } + + s, err := PackSiblings(t.hashFunction, siblings) + if err != nil { + return nil, nil, nil, false, err + } + + leafK, leafV := ReadLeafValue(value) + if !bytes.Equal(k, leafK) { + // key not in tree, proof of non-existence + return leafK, leafV, s, false, nil + } + + return leafK, leafV, s, true, nil +} + +// PackSiblings packs the siblings into a byte array. +// [ 2 byte | 2 byte | L bytes | S * N bytes ] +// [ full length | bitmap length (L) | bitmap | N non-zero siblings ] +// Where the bitmap indicates if the sibling is 0 or a value from the siblings +// array. And S is the size of the output of the hash function used for the +// Tree. The 2 2-byte that define the full length and bitmap length, are +// encoded in little-endian. +func PackSiblings(hashFunc HashFunction, siblings [][]byte) ([]byte, error) { + var b []byte + var bitmap []bool + emptySibling := make([]byte, hashFunc.Len()) + for i := 0; i < len(siblings); i++ { + if bytes.Equal(siblings[i], emptySibling) { + bitmap = append(bitmap, false) + } else { + bitmap = append(bitmap, true) + b = append(b, siblings[i]...) + } + } + + bitmapBytes := bitmapToBytes(bitmap) + l := len(bitmapBytes) + if l > maxUint16 { + return nil, fmt.Errorf("PackSiblings: bitmapBytes length > %v", maxUint16) + } + + fullLen := 4 + l + len(b) + if fullLen > maxUint16 { + return nil, fmt.Errorf("PackSiblings: fullLen > %v", maxUint16) + } + res := make([]byte, fullLen) + binary.LittleEndian.PutUint16(res[0:2], uint16(fullLen)) // set full length + binary.LittleEndian.PutUint16(res[2:4], uint16(l)) // set the bitmapBytes length + copy(res[4:4+l], bitmapBytes) + copy(res[4+l:], b) + return res, nil +} + +// UnpackSiblings unpacks the siblings from a byte array. +func UnpackSiblings(hashFunc HashFunction, b []byte) ([][]byte, error) { + // to prevent runtime slice out of bounds error check if the length of the + // rest of the slice is at least equal to the encoded full length value + if len(b) < 4 { + return nil, fmt.Errorf("no packed siblings provided") + } + + fullLen := binary.LittleEndian.Uint16(b[0:2]) + if len(b) != int(fullLen) { + return nil, fmt.Errorf("expected len: %d, current len: %d", fullLen, len(b)) + } + + l := binary.LittleEndian.Uint16(b[2:4]) // bitmap bytes length + // to prevent runtime slice out of bounds error check if the length of the + // rest of the slice is at least equal to the encoded bitmap length value + if len(b) < int(4+l) { + return nil, fmt.Errorf("expected len: %d, current len: %d", 4+l, len(b)) + } + bitmapBytes := b[4 : 4+l] + bitmap := bytesToBitmap(bitmapBytes) + siblingsBytes := b[4+l:] + // to prevent a runtime slice out of bounds error, check if the length of + // the siblings slice is a multiple of hashFunc length, because the + // following loop will iterate over it in steps of that length. + if len(siblingsBytes)%hashFunc.Len() != 0 { + return nil, fmt.Errorf("bad formated siblings") + } + iSibl := 0 + emptySibl := make([]byte, hashFunc.Len()) + var siblings [][]byte + for i := 0; i < len(bitmap); i++ { + if iSibl >= len(siblingsBytes) { + break + } + if bitmap[i] { + siblings = append(siblings, siblingsBytes[iSibl:iSibl+hashFunc.Len()]) + iSibl += hashFunc.Len() + } else { + siblings = append(siblings, emptySibl) + } + } + return siblings, nil +} + +func bitmapToBytes(bitmap []bool) []byte { + bitmapBytesLen := int(math.Ceil(float64(len(bitmap)) / 8)) + b := make([]byte, bitmapBytesLen) + for i := 0; i < len(bitmap); i++ { + if bitmap[i] { + b[i/8] |= 1 << (i % 8) + } + } + return b +} + +func bytesToBitmap(b []byte) []bool { + var bitmap []bool + for i := 0; i < len(b); i++ { + for j := 0; j < 8; j++ { + bitmap = append(bitmap, b[i]&(1< 0) + } + } + return bitmap +} + +// CheckProof verifies the given proof. The proof verification depends on the +// HashFunction passed as parameter. +func CheckProof(hashFunc HashFunction, k, v, root, packedSiblings []byte) (bool, error) { + siblings, err := UnpackSiblings(hashFunc, packedSiblings) + if err != nil { + return false, err + } + + keyPath := make([]byte, int(math.Ceil(float64(len(siblings))/float64(8)))) + copy(keyPath, k) + + key, _, err := newLeafValue(hashFunc, k, v) + if err != nil { + return false, err + } + + path := getPath(len(siblings), keyPath) + for i := len(siblings) - 1; i >= 0; i-- { + if path[i] { + key, _, err = newIntermediate(hashFunc, siblings[i], key) + if err != nil { + return false, err + } + } else { + key, _, err = newIntermediate(hashFunc, key, siblings[i]) + if err != nil { + return false, err + } + } + } + if bytes.Equal(key, root) { + return true, nil + } + return false, nil +} diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 6947c374b..7df8850fb 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -17,8 +17,6 @@ import ( "encoding/hex" "fmt" "io" - "math" - "runtime" "sync" "go.vocdoni.io/dvote/db" @@ -142,8 +140,12 @@ func NewTreeWithTx(wTx db.WriteTx, cfg Config) (*Tree, error) { cfg.ThresholdNLeafs = DefaultThresholdNLeafs } - t := Tree{db: cfg.Database, maxLevels: cfg.MaxLevels, - thresholdNLeafs: cfg.ThresholdNLeafs, hashFunction: cfg.HashFunction} + t := Tree{ + db: cfg.Database, + maxLevels: cfg.MaxLevels, + thresholdNLeafs: cfg.ThresholdNLeafs, + hashFunction: cfg.HashFunction, + } t.emptyHash = make([]byte, t.hashFunction.Len()) // empty var err error @@ -207,320 +209,6 @@ type Invalid struct { Error error } -// AddBatch adds a batch of key-values to the Tree. Returns an array containing -// the indexes of the keys failed to add. Supports empty values as input -// parameters, which is equivalent to 0 valued byte array. -func (t *Tree) AddBatch(keys, values [][]byte) ([]Invalid, error) { - wTx := t.db.WriteTx() - defer wTx.Discard() - - invalids, err := t.AddBatchWithTx(wTx, keys, values) - if err != nil { - return invalids, err - } - return invalids, wTx.Commit() -} - -// AddBatchWithTx does the same than the AddBatch method, but allowing to pass -// the db.WriteTx that is used. The db.WriteTx will not be committed inside -// this method. -func (t *Tree) AddBatchWithTx(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { - t.Lock() - defer t.Unlock() - - if !t.editable() { - return nil, ErrSnapshotNotEditable - } - - e := []byte{} - // equal the number of keys & values - if len(keys) > len(values) { - // add missing values - for i := len(values); i < len(keys); i++ { - values = append(values, e) - } - } else if len(keys) < len(values) { - // crop extra values - values = values[:len(keys)] - } - - nLeafs, err := t.GetNLeafsWithTx(wTx) - if err != nil { - return nil, err - } - if nLeafs > t.thresholdNLeafs { - return t.addBatchInDisk(wTx, keys, values) - } - return t.addBatchInMemory(wTx, keys, values) -} - -func (t *Tree) addBatchInDisk(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { - nCPU := flp2(runtime.NumCPU()) - if nCPU == 1 || len(keys) < nCPU { - var invalids []Invalid - for i := 0; i < len(keys); i++ { - if err := t.addWithTx(wTx, keys[i], values[i]); err != nil { - invalids = append(invalids, Invalid{i, err}) - } - } - return invalids, nil - } - - kvs, invalids, err := keysValuesToKvs(t.maxLevels, keys, values) - if err != nil { - return nil, err - } - - buckets := splitInBuckets(kvs, nCPU) - - root, err := t.RootWithTx(wTx) - if err != nil { - return nil, err - } - - l := int(math.Log2(float64(nCPU))) - subRoots, err := t.getSubRootsAtLevel(wTx, root, l+1) - if err != nil { - return nil, err - } - if len(subRoots) != nCPU { - // Already populated Tree but Unbalanced. - - // add one key at each bucket, and then continue with the flow - for i := 0; i < len(buckets); i++ { - // add one leaf of the bucket, if there is an error when - // adding the k-v, try to add the next one of the bucket - // (until one is added) - inserted := -1 - for j := 0; j < len(buckets[i]); j++ { - if newRoot, err := t.add(wTx, root, 0, - buckets[i][j].k, buckets[i][j].v); err == nil { - inserted = j - root = newRoot - break - } - } - - // remove the inserted element from buckets[i] - if inserted != -1 { - buckets[i] = append(buckets[i][:inserted], buckets[i][inserted+1:]...) - } - } - subRoots, err = t.getSubRootsAtLevel(wTx, root, l+1) - if err != nil { - return nil, err - } - } - - if len(subRoots) != nCPU { - return nil, fmt.Errorf("this error should not be reached."+ - " len(subRoots) != nCPU, len(subRoots)=%d, nCPU=%d."+ - " Please report it in a new issue:"+ - " https://go.vocdoni.io/dvote/tree/arbo/issues/new", len(subRoots), nCPU) - } - - invalidsInBucket := make([][]Invalid, nCPU) - txs := make([]db.WriteTx, nCPU) - for i := 0; i < nCPU; i++ { - txs[i] = t.db.WriteTx() - err := txs[i].Apply(wTx) - if err != nil { - return nil, err - } - } - - var wg sync.WaitGroup - wg.Add(nCPU) - for i := 0; i < nCPU; i++ { - go func(cpu int) { - // use different wTx for each cpu, after once all - // are done, iter over the cpuWTxs and copy their - // content into the main wTx - for j := 0; j < len(buckets[cpu]); j++ { - newSubRoot, err := t.add(txs[cpu], subRoots[cpu], - l, buckets[cpu][j].k, buckets[cpu][j].v) - if err != nil { - invalidsInBucket[cpu] = append(invalidsInBucket[cpu], - Invalid{buckets[cpu][j].pos, err}) - continue - } - // if there has not been errors, set the new subRoots[cpu] - subRoots[cpu] = newSubRoot - } - wg.Done() - }(i) - } - wg.Wait() - - for i := 0; i < nCPU; i++ { - if err := wTx.Apply(txs[i]); err != nil { - return nil, err - } - txs[i].Discard() - } - - for i := 0; i < len(invalidsInBucket); i++ { - invalids = append(invalids, invalidsInBucket[i]...) - } - - newRoot, err := t.upFromSubRoots(wTx, subRoots) - if err != nil { - return nil, err - } - - // update dbKeyNLeafs - if err := t.SetRootWithTx(wTx, newRoot); err != nil { - return nil, err - } - - // update nLeafs - if err := t.incNLeafs(wTx, len(keys)-len(invalids)); err != nil { - return nil, err - } - - return invalids, nil -} - -func (t *Tree) upFromSubRoots(wTx db.WriteTx, subRoots [][]byte) ([]byte, error) { - // is a method of Tree just to get access to t.hashFunction and - // t.emptyHash. - - // go up from subRoots to up, storing nodes in the given WriteTx - // once up at the root, store it in the WriteTx using the dbKeyRoot - if len(subRoots) == 1 { - return subRoots[0], nil - } - // get the subRoots values to know the node types of each subRoot - nodeTypes := make([]byte, len(subRoots)) - for i := 0; i < len(subRoots); i++ { - if bytes.Equal(subRoots[i], t.emptyHash) { - nodeTypes[i] = PrefixValueEmpty - continue - } - v, err := wTx.Get(subRoots[i]) - if err != nil { - return nil, err - } - nodeTypes[i] = v[0] - } - - var newSubRoots [][]byte - for i := 0; i < len(subRoots); i += 2 { - if (bytes.Equal(subRoots[i], t.emptyHash) && bytes.Equal(subRoots[i+1], t.emptyHash)) || - (nodeTypes[i] == PrefixValueLeaf && bytes.Equal(subRoots[i+1], t.emptyHash)) { - // when both sub nodes are empty, the parent is also empty - // or - // when 1st sub node is a leaf but the 2nd is empty, the - // leaf is used as 'parent' - - newSubRoots = append(newSubRoots, subRoots[i]) - continue - } - if bytes.Equal(subRoots[i], t.emptyHash) && nodeTypes[i+1] == PrefixValueLeaf { - // when 2nd sub node is a leaf but the 1st is empty, - // the leaf is used as 'parent' - newSubRoots = append(newSubRoots, subRoots[i+1]) - continue - } - - k, v, err := t.newIntermediate(subRoots[i], subRoots[i+1]) - if err != nil { - return nil, err - } - // store k-v to db - if err = wTx.Set(k, v); err != nil { - return nil, err - } - newSubRoots = append(newSubRoots, k) - } - - return t.upFromSubRoots(wTx, newSubRoots) -} - -func (t *Tree) getSubRootsAtLevel(rTx db.Reader, root []byte, l int) ([][]byte, error) { - // go at level l and return each node key, where each node key is the - // subRoot of the subTree that starts there - - var subRoots [][]byte - err := t.iterWithStop(rTx, root, 0, func(currLvl int, k, v []byte) bool { - if currLvl == l && !bytes.Equal(k, t.emptyHash) { - subRoots = append(subRoots, k) - } - if currLvl >= l { - return true // to stop the iter from going down - } - return false - }) - - return subRoots, err -} - -func (t *Tree) addBatchInMemory(wTx db.WriteTx, keys, values [][]byte) ([]Invalid, error) { - vt, err := t.loadVT(wTx) - if err != nil { - return nil, err - } - - invalids, err := vt.addBatch(keys, values) - if err != nil { - return nil, err - } - - // once the VirtualTree is build, compute the hashes - pairs, err := vt.computeHashes() - if err != nil { - // currently invalids in computeHashes are not counted, - // but should not be needed, as if there is an error there is - // nothing stored in the db and the error is returned - return nil, err - } - - // store pairs in db - for i := 0; i < len(pairs); i++ { - if err := wTx.Set(pairs[i][0], pairs[i][1]); err != nil { - return nil, err - } - } - - // store root (from the vt) to db - if vt.root != nil { - if err := wTx.Set(dbKeyRoot, vt.root.h); err != nil { - return nil, err - } - } - - // update nLeafs - if err := t.incNLeafs(wTx, len(keys)-len(invalids)); err != nil { - return nil, err - } - - return invalids, nil -} - -// loadVT loads a new virtual tree (vt) from the current Tree, which contains -// the same leafs. -func (t *Tree) loadVT(rTx db.Reader) (vt, error) { - vt := newVT(t.maxLevels, t.hashFunction) - vt.params.dbg = t.dbg - var callbackErr error - err := t.IterateWithStopWithTx(rTx, nil, func(_ int, k, v []byte) bool { - if v[0] != PrefixValueLeaf { - return false - } - leafK, leafV := ReadLeafValue(v) - if err := vt.add(0, leafK, leafV); err != nil { - callbackErr = err - return true - } - return false - }) - if callbackErr != nil { - return vt, callbackErr - } - - return vt, err -} - // Add inserts the key-value into the Tree. If the inputs come from a // *big.Int, is expected that are represented by a Little-Endian byte array // (for circom compatibility). @@ -565,45 +253,7 @@ func (t *Tree) addWithTx(wTx db.WriteTx, k, v []byte) error { return err } // update nLeafs - if err = t.incNLeafs(wTx, 1); err != nil { - return err - } - return nil -} - -// keyPathFromKey returns the keyPath and checks that the key is not bigger -// than maximum key length for the tree maxLevels size. -// This is because if the key bits length is bigger than the maxLevels of the -// tree, two different keys that their difference is at the end, will collision -// in the same leaf of the tree (at the max depth). -func keyPathFromKey(maxLevels int, k []byte) ([]byte, error) { - maxKeyLen := int(math.Ceil(float64(maxLevels) / float64(8))) - if len(k) > maxKeyLen { - return nil, fmt.Errorf("len(k) can not be bigger than ceil(maxLevels/8), where"+ - " len(k): %d, maxLevels: %d, max key len=ceil(maxLevels/8): %d. Might need"+ - " a bigger tree depth (maxLevels>=%d) in order to input keys of length %d", - len(k), maxLevels, maxKeyLen, len(k)*8, len(k)) - } - keyPath := make([]byte, maxKeyLen) - copy(keyPath, k) - return keyPath, nil -} - -// checkKeyValueLen checks the key length and value length. This method is used -// when adding single leafs and also when adding a batch. The limits of lengths -// used are derived from the encoding of tree dumps: 1 byte to define the -// length of the keys (2^8-1 bytes length)), and 2 bytes to define the length -// of the values (2^16-1 bytes length). -func checkKeyValueLen(k, v []byte) error { - if len(k) > maxUint8 { - return fmt.Errorf("len(k)=%v, can not be bigger than %v", - len(k), maxUint8) - } - if len(v) > maxUint16 { - return fmt.Errorf("len(v)=%v, can not be bigger than %v", - len(v), maxUint16) - } - return nil + return t.incNLeafs(wTx, 1) } func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byte, error) { @@ -618,10 +268,9 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt path := getPath(t.maxLevels, keyPath) // go down to the leaf - var siblings [][]byte - intermediates := make([][]byte, t.maxLevels) + var siblings, intermediates [][]byte - _, _, siblings, intermediates, err = t.down(wTx, k, root, siblings, intermediates, path, fromLvl, false) + _, _, siblings, err = t.down(wTx, k, root, siblings, &intermediates, path, fromLvl, false) if err != nil { return nil, err } @@ -630,6 +279,7 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt if err != nil { return nil, err } + if err := wTx.Set(leafKey, leafValue); err != nil { return nil, err } @@ -639,36 +289,16 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt // return the leafKey as root return leafKey, nil } - root, err = t.up(wTx, intermediates, leafKey, siblings, path, len(siblings)-1, fromLvl) + root, err = t.up(wTx, leafKey, siblings, path, len(siblings)-1, fromLvl) if err != nil { return nil, err } - return root, nil -} - -// downVirtually is used when in a leaf already exists, and a new leaf which -// shares the path until the existing leaf is being added -func (t *Tree) downVirtually(siblings [][]byte, oldKey, newKey []byte, oldPath, - newPath []bool, currLvl int) ([][]byte, error) { - var err error - if currLvl > t.maxLevels-1 { - return nil, ErrMaxVirtualLevel + if err := deleteNodes(wTx, intermediates); err != nil { + return root, fmt.Errorf("error deleting orphan intermediate nodes: %v", err) } - if oldPath[currLvl] == newPath[currLvl] { - siblings = append(siblings, t.emptyHash) - - siblings, err = t.downVirtually(siblings, oldKey, newKey, oldPath, newPath, currLvl+1) - if err != nil { - return nil, err - } - return siblings, nil - } - // reached the divergence - siblings = append(siblings, oldKey) - - return siblings, nil + return root, nil } func (t *Tree) newLeafValue(k, v []byte) ([]byte, []byte, error) { @@ -676,43 +306,6 @@ func (t *Tree) newLeafValue(k, v []byte) ([]byte, []byte, error) { return newLeafValue(t.hashFunction, k, v) } -// newLeafValue takes a key & value from a leaf, and computes the leaf hash, -// which is used as the leaf key. And the value is the concatenation of the -// inputted key & value. The output of this function is used as key-value to -// store the leaf in the DB. -// [ 1 byte | 1 byte | N bytes | M bytes ] -// [ type of node | length of key | key | value ] -func newLeafValue(hashFunc HashFunction, k, v []byte) ([]byte, []byte, error) { - if err := checkKeyValueLen(k, v); err != nil { - return nil, nil, err - } - leafKey, err := hashFunc.Hash(k, v, []byte{1}) - if err != nil { - return nil, nil, err - } - var leafValue []byte - leafValue = append(leafValue, byte(PrefixValueLeaf)) - leafValue = append(leafValue, byte(len(k))) - leafValue = append(leafValue, k...) - leafValue = append(leafValue, v...) - return leafKey, leafValue, nil -} - -// ReadLeafValue reads from a byte array the leaf key & value -func ReadLeafValue(b []byte) ([]byte, []byte) { - if len(b) < PrefixValueLen { - return []byte{}, []byte{} - } - - kLen := b[1] - if len(b) < PrefixValueLen+int(kLen) { - return []byte{}, []byte{} - } - k := b[PrefixValueLen : PrefixValueLen+kLen] - v := b[PrefixValueLen+kLen:] - return k, v -} - func (t *Tree) newIntermediate(l, r []byte) ([]byte, []byte, error) { t.dbg.incHash() return newIntermediate(t.hashFunction, l, r) @@ -752,36 +345,49 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { return err } - var siblings [][]byte - intermediates := make([][]byte, t.maxLevels) - - _, valueAtBottom, siblings, intermediates, err := t.down(wTx, k, root, siblings, intermediates, path, 0, true) + var siblings, intermediates [][]byte + oldLeafKey, valueAtBottom, siblings, err := t.down(wTx, k, root, siblings, &intermediates, path, 0, true) if err != nil { return err } + + // check if the key is actually the same oldKey, _ := ReadLeafValue(valueAtBottom) if !bytes.Equal(oldKey, k) { return ErrKeyNotFound } + // delete the old leaf key + if err := wTx.Delete(oldLeafKey); err != nil { + return fmt.Errorf("error deleting old leaf on update: %w", err) + } + + // compute the new leaf key and value leafKey, leafValue, err := t.newLeafValue(k, v) if err != nil { return err } + // add the new leaf key if err := wTx.Set(leafKey, leafValue); err != nil { return err } // go up to the root + if len(siblings) == 0 { return t.setRoot(wTx, leafKey) } - root, err = t.up(wTx, intermediates, leafKey, siblings, path, len(siblings)-1, 0) + + root, err = t.up(wTx, leafKey, siblings, path, len(siblings)-1, 0) if err != nil { return err } + if err := deleteNodes(wTx, intermediates); err != nil { + return fmt.Errorf("error deleting orphan intermediate nodes: %v", err) + } + // store root to db return t.setRoot(wTx, root) } @@ -819,11 +425,10 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { } // Navigate down the tree to find the key and collect siblings. - var siblings [][]byte - intermediates := make([][]byte, t.maxLevels) + var siblings, intermediates [][]byte path := getPath(t.maxLevels, k) - leafKey, _, siblings, intermediates, err := t.down(wTx, k, root, siblings, intermediates, path, 0, true) + leafKey, _, siblings, err := t.down(wTx, k, root, siblings, &intermediates, path, 0, true) if err != nil { if err == ErrKeyNotFound { // Key not found, nothing to delete. @@ -832,9 +437,16 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return err } - // delete the leaf key - if err := wTx.Delete(leafKey); err != nil { - return fmt.Errorf("error deleting key %x: %w", leafKey, err) + // if the neighbor is not empty, set the leaf key to the empty hash + if !bytes.Equal(siblings[len(siblings)-1], t.emptyHash) { + if err := wTx.Set(leafKey, t.emptyHash); err != nil { + return fmt.Errorf("error setting leaf key %x to empty hash: %w", leafKey, err) + } + } else { + // else delete the leaf key + if err := wTx.Delete(leafKey); err != nil { + return fmt.Errorf("error deleting leaf key %x: %w", leafKey, err) + } } // Navigate back up the tree, updating the intermediate nodes. @@ -842,7 +454,7 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { // The tree is empty after deletion. return t.setRoot(wTx, t.emptyHash) } - newRoot, err := t.up(wTx, intermediates, t.emptyHash, siblings, path, len(siblings)-1, 0) + newRoot, err := t.up(wTx, t.emptyHash, siblings, path, len(siblings)-1, 0) if err != nil { return err } @@ -852,165 +464,12 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return err } - // Update the number of leaves. - return t.decNLeafs(wTx, 1) -} - -// decNLeafs decreases the number of leaves in the tree. -func (t *Tree) decNLeafs(wTx db.WriteTx, n int) error { - nLeafs, err := t.GetNLeafsWithTx(wTx) - if err != nil { - return err - } - return t.setNLeafs(wTx, nLeafs-n) -} - -// GenProof generates a MerkleTree proof for the given key. The leaf value is -// returned, together with the packed siblings of the proof, and a boolean -// parameter that indicates if the proof is of existence (true) or not (false). -func (t *Tree) GenProof(k []byte) ([]byte, []byte, []byte, bool, error) { - return t.GenProofWithTx(t.db, k) -} - -// GenProofWithTx does the same than the GenProof method, but allowing to pass -// the db.ReadTx that is used. -func (t *Tree) GenProofWithTx(rTx db.Reader, k []byte) ([]byte, []byte, []byte, bool, error) { - keyPath, err := keyPathFromKey(t.maxLevels, k) - if err != nil { - return nil, nil, nil, false, err - } - path := getPath(t.maxLevels, keyPath) - - root, err := t.RootWithTx(rTx) - if err != nil { - return nil, nil, nil, false, err - } - - // go down to the leaf - var siblings [][]byte - intermediates := make([][]byte, t.maxLevels) - - _, value, siblings, intermediates, err := t.down(rTx, k, root, siblings, intermediates, path, 0, true) - if err != nil { - return nil, nil, nil, false, err + if err := deleteNodes(wTx, intermediates); err != nil { + return fmt.Errorf("error deleting orphan intermediate nodes: %v", err) } - s, err := PackSiblings(t.hashFunction, siblings) - if err != nil { - return nil, nil, nil, false, err - } - - leafK, leafV := ReadLeafValue(value) - if !bytes.Equal(k, leafK) { - // key not in tree, proof of non-existence - return leafK, leafV, s, false, nil - } - - return leafK, leafV, s, true, nil -} - -// PackSiblings packs the siblings into a byte array. -// [ 2 byte | 2 byte | L bytes | S * N bytes ] -// [ full length | bitmap length (L) | bitmap | N non-zero siblings ] -// Where the bitmap indicates if the sibling is 0 or a value from the siblings -// array. And S is the size of the output of the hash function used for the -// Tree. The 2 2-byte that define the full length and bitmap length, are -// encoded in little-endian. -func PackSiblings(hashFunc HashFunction, siblings [][]byte) ([]byte, error) { - var b []byte - var bitmap []bool - emptySibling := make([]byte, hashFunc.Len()) - for i := 0; i < len(siblings); i++ { - if bytes.Equal(siblings[i], emptySibling) { - bitmap = append(bitmap, false) - } else { - bitmap = append(bitmap, true) - b = append(b, siblings[i]...) - } - } - - bitmapBytes := bitmapToBytes(bitmap) - l := len(bitmapBytes) - if l > maxUint16 { - return nil, fmt.Errorf("PackSiblings: bitmapBytes length > %v", maxUint16) - } - - fullLen := 4 + l + len(b) - if fullLen > maxUint16 { - return nil, fmt.Errorf("PackSiblings: fullLen > %v", maxUint16) - } - res := make([]byte, fullLen) - binary.LittleEndian.PutUint16(res[0:2], uint16(fullLen)) // set full length - binary.LittleEndian.PutUint16(res[2:4], uint16(l)) // set the bitmapBytes length - copy(res[4:4+l], bitmapBytes) - copy(res[4+l:], b) - return res, nil -} - -// UnpackSiblings unpacks the siblings from a byte array. -func UnpackSiblings(hashFunc HashFunction, b []byte) ([][]byte, error) { - // to prevent runtime slice out of bounds error check if the length of the - // rest of the slice is at least equal to the encoded full length value - if len(b) < 4 { - return nil, fmt.Errorf("no packed siblings provided") - } - - fullLen := binary.LittleEndian.Uint16(b[0:2]) - if len(b) != int(fullLen) { - return nil, fmt.Errorf("expected len: %d, current len: %d", fullLen, len(b)) - } - - l := binary.LittleEndian.Uint16(b[2:4]) // bitmap bytes length - // to prevent runtime slice out of bounds error check if the length of the - // rest of the slice is at least equal to the encoded bitmap length value - if len(b) < int(4+l) { - return nil, fmt.Errorf("expected len: %d, current len: %d", 4+l, len(b)) - } - bitmapBytes := b[4 : 4+l] - bitmap := bytesToBitmap(bitmapBytes) - siblingsBytes := b[4+l:] - // to prevent a runtime slice out of bounds error, check if the length of - // the siblings slice is a multiple of hashFunc length, because the - // following loop will iterate over it in steps of that length. - if len(siblingsBytes)%hashFunc.Len() != 0 { - return nil, fmt.Errorf("bad formated siblings") - } - iSibl := 0 - emptySibl := make([]byte, hashFunc.Len()) - var siblings [][]byte - for i := 0; i < len(bitmap); i++ { - if iSibl >= len(siblingsBytes) { - break - } - if bitmap[i] { - siblings = append(siblings, siblingsBytes[iSibl:iSibl+hashFunc.Len()]) - iSibl += hashFunc.Len() - } else { - siblings = append(siblings, emptySibl) - } - } - return siblings, nil -} - -func bitmapToBytes(bitmap []bool) []byte { - bitmapBytesLen := int(math.Ceil(float64(len(bitmap)) / 8)) - b := make([]byte, bitmapBytesLen) - for i := 0; i < len(bitmap); i++ { - if bitmap[i] { - b[i/8] |= 1 << (i % 8) - } - } - return b -} - -func bytesToBitmap(b []byte) []bool { - var bitmap []bool - for i := 0; i < len(b); i++ { - for j := 0; j < 8; j++ { - bitmap = append(bitmap, b[i]&(1< 0) - } - } - return bitmap + // Update the number of leaves. + return t.decNLeafs(wTx, 1) } // Get returns the value in the Tree for a given key. If the key is not found, @@ -1038,10 +497,9 @@ func (t *Tree) GetWithTx(rTx db.Reader, k []byte) ([]byte, []byte, error) { } // go down to the leaf - var siblings [][]byte - intermediates := make([][]byte, t.maxLevels) + var siblings, intermediates [][]byte - _, value, _, intermediates, err := t.down(rTx, k, root, siblings, intermediates, path, 0, true) + _, value, _, err := t.down(rTx, k, root, siblings, &intermediates, path, 0, true) if err != nil { return nil, nil, err } @@ -1053,40 +511,13 @@ func (t *Tree) GetWithTx(rTx db.Reader, k []byte) ([]byte, []byte, error) { return leafK, leafV, nil } -// CheckProof verifies the given proof. The proof verification depends on the -// HashFunction passed as parameter. -func CheckProof(hashFunc HashFunction, k, v, root, packedSiblings []byte) (bool, error) { - siblings, err := UnpackSiblings(hashFunc, packedSiblings) - if err != nil { - return false, err - } - - keyPath := make([]byte, int(math.Ceil(float64(len(siblings))/float64(8)))) - copy(keyPath, k) - - key, _, err := newLeafValue(hashFunc, k, v) +// decNLeafs decreases the number of leaves in the tree. +func (t *Tree) decNLeafs(wTx db.WriteTx, n int) error { + nLeafs, err := t.GetNLeafsWithTx(wTx) if err != nil { - return false, err - } - - path := getPath(len(siblings), keyPath) - for i := len(siblings) - 1; i >= 0; i-- { - if path[i] { - key, _, err = newIntermediate(hashFunc, siblings[i], key) - if err != nil { - return false, err - } - } else { - key, _, err = newIntermediate(hashFunc, key, siblings[i]) - if err != nil { - return false, err - } - } - } - if bytes.Equal(key, root) { - return true, nil + return err } - return false, nil + return t.setNLeafs(wTx, nLeafs-n) } func (t *Tree) incNLeafs(wTx db.WriteTx, nLeafs int) error { @@ -1101,10 +532,7 @@ func (t *Tree) incNLeafs(wTx db.WriteTx, nLeafs int) error { func (*Tree) setNLeafs(wTx db.WriteTx, nLeafs int) error { b := make([]byte, 8) binary.LittleEndian.PutUint64(b, uint64(nLeafs)) - if err := wTx.Set(dbKeyNLeafs, b); err != nil { - return err - } - return nil + return wTx.Set(dbKeyNLeafs, b) } // GetNLeafs returns the number of Leafs of the Tree. diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 7bfe37295..8daa79126 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -994,7 +994,7 @@ func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) { func TestDiskSizeBench(t *testing.T) { c := qt.New(t) - nLeafs := 20 + nLeafs := 2000 printTestContext("TestDiskSizeBench: ", nLeafs, "Blake2b", "pebble") // prepare inputs @@ -1044,8 +1044,6 @@ func TestDiskSizeBench(t *testing.T) { printRes(" Disk size (add)", fmt.Sprintf("%d MiB", size/(1024*1024))) printRes(" DB items", fmt.Sprintf("%d", countDBitems())) - tree.PrintGraphviz(nil) - // delete the leafs start = time.Now() tx = tree.db.WriteTx() @@ -1062,10 +1060,6 @@ func TestDiskSizeBench(t *testing.T) { printRes(" Disk size (delete)", fmt.Sprintf("%d MiB", size/(1024*1024))) printRes(" DB items", fmt.Sprintf("%d", countDBitems())) - tree.PrintGraphviz(nil) - - return - // add the leafs deleted again start = time.Now() tx = tree.db.WriteTx() diff --git a/tree/arbo/utils.go b/tree/arbo/utils.go index 973b8ade9..c183ff315 100644 --- a/tree/arbo/utils.go +++ b/tree/arbo/utils.go @@ -1,7 +1,11 @@ package arbo import ( + "fmt" + "math" "math/big" + + "go.vocdoni.io/dvote/db" ) // SwapEndianness swaps the order of the bytes in the byte slice. @@ -26,3 +30,85 @@ func BigIntToBytes(blen int, bi *big.Int) []byte { func BytesToBigInt(b []byte) *big.Int { return new(big.Int).SetBytes(SwapEndianness(b)) } + +// newLeafValue takes a key & value from a leaf, and computes the leaf hash, +// which is used as the leaf key. And the value is the concatenation of the +// inputted key & value. The output of this function is used as key-value to +// store the leaf in the DB. +// [ 1 byte | 1 byte | N bytes | M bytes ] +// [ type of node | length of key | key | value ] +func newLeafValue(hashFunc HashFunction, k, v []byte) ([]byte, []byte, error) { + if err := checkKeyValueLen(k, v); err != nil { + return nil, nil, err + } + leafKey, err := hashFunc.Hash(k, v, []byte{1}) + if err != nil { + return nil, nil, err + } + var leafValue []byte + leafValue = append(leafValue, byte(PrefixValueLeaf)) + leafValue = append(leafValue, byte(len(k))) + leafValue = append(leafValue, k...) + leafValue = append(leafValue, v...) + return leafKey, leafValue, nil +} + +// ReadLeafValue reads from a byte array the leaf key & value +func ReadLeafValue(b []byte) ([]byte, []byte) { + if len(b) < PrefixValueLen { + return []byte{}, []byte{} + } + + kLen := b[1] + if len(b) < PrefixValueLen+int(kLen) { + return []byte{}, []byte{} + } + k := b[PrefixValueLen : PrefixValueLen+kLen] + v := b[PrefixValueLen+kLen:] + return k, v +} + +// keyPathFromKey returns the keyPath and checks that the key is not bigger +// than maximum key length for the tree maxLevels size. +// This is because if the key bits length is bigger than the maxLevels of the +// tree, two different keys that their difference is at the end, will collision +// in the same leaf of the tree (at the max depth). +func keyPathFromKey(maxLevels int, k []byte) ([]byte, error) { + maxKeyLen := int(math.Ceil(float64(maxLevels) / float64(8))) + if len(k) > maxKeyLen { + return nil, fmt.Errorf("len(k) can not be bigger than ceil(maxLevels/8), where"+ + " len(k): %d, maxLevels: %d, max key len=ceil(maxLevels/8): %d. Might need"+ + " a bigger tree depth (maxLevels>=%d) in order to input keys of length %d", + len(k), maxLevels, maxKeyLen, len(k)*8, len(k)) + } + keyPath := make([]byte, maxKeyLen) + copy(keyPath, k) + return keyPath, nil +} + +// checkKeyValueLen checks the key length and value length. This method is used +// when adding single leafs and also when adding a batch. The limits of lengths +// used are derived from the encoding of tree dumps: 1 byte to define the +// length of the keys (2^8-1 bytes length)), and 2 bytes to define the length +// of the values (2^16-1 bytes length). +func checkKeyValueLen(k, v []byte) error { + if len(k) > maxUint8 { + return fmt.Errorf("len(k)=%v, can not be bigger than %v", + len(k), maxUint8) + } + if len(v) > maxUint16 { + return fmt.Errorf("len(v)=%v, can not be bigger than %v", + len(v), maxUint16) + } + return nil +} + +// deleteNodes removes the nodes in the keys slice from the database. +func deleteNodes(wTx db.WriteTx, keys [][]byte) error { + for _, k := range keys { + if err := wTx.Delete(k); err != nil { + return err + } + } + return nil +} From 317f1bb38da2766fe3bfb65163d2b6e814b6efec Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 30 Aug 2023 16:29:06 +0200 Subject: [PATCH 09/15] arbo: reorder neighbour nodes on delete When a leaf of the tree is removed, if the neighbour child is not empty, it should be moved to the upper level where there are no other leaves. Add tests to verify the operation. --- tree/arbo/addbatch_test.go | 4 +- tree/arbo/navigate.go | 65 ++++++++++++++++ tree/arbo/tree.go | 36 +++++---- tree/arbo/tree_test.go | 151 +++++++++++++++++++++++++++++++++++-- 4 files changed, 235 insertions(+), 21 deletions(-) diff --git a/tree/arbo/addbatch_test.go b/tree/arbo/addbatch_test.go index 2c21fc0c8..6c77694e8 100644 --- a/tree/arbo/addbatch_test.go +++ b/tree/arbo/addbatch_test.go @@ -13,9 +13,7 @@ import ( "time" qt "github.com/frankban/quicktest" - "go.vocdoni.io/dvote/db" "go.vocdoni.io/dvote/db/metadb" - "go.vocdoni.io/dvote/db/pebbledb" ) var debug = os.Getenv("LOG_LEVEL") == "debug" @@ -978,6 +976,7 @@ func TestAddKeysWithEmptyValues(t *testing.T) { c.Check(verif, qt.IsFalse) } +/* func TestAddBatchThresholdInDisk(t *testing.T) { c := qt.New(t) @@ -1046,6 +1045,7 @@ func TestAddBatchThresholdInDisk(t *testing.T) { err = tree3.Add(k, v) c.Assert(err, qt.IsNil) } +*/ func initTestUpFromSubRoots(c *qt.C) (*Tree, *Tree) { database1 := metadb.NewTest(c) diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go index 16940f12d..c41d4e226 100644 --- a/tree/arbo/navigate.go +++ b/tree/arbo/navigate.go @@ -180,9 +180,74 @@ func ReadIntermediateChilds(b []byte) ([]byte, []byte) { } func getPath(numLevels int, k []byte) []bool { + requiredLen := (numLevels + 7) / 8 // Calculate the ceil value of numLevels/8 + if len(k) < requiredLen { + // The provided key is shorter than expected for the given number of levels. + // Handle this case appropriately, either by returning an error or by handling it in some other way. + panic(fmt.Sprintf("key length is too short: expected at least %d bytes, got %d bytes", requiredLen, len(k))) + } + path := make([]bool, numLevels) for n := 0; n < numLevels; n++ { path[n] = k[n/8]&(1<<(n%8)) != 0 } return path } + +// getLeavesFromSubPath navigates the Merkle tree from a given node key +// and collects all the leaves found within its subpath. The function can +// start from an intermediate node or a leaf itself. +// Returns the list of keys and values of the leaves found. +func (t *Tree) getLeavesFromSubPath(rTx db.Reader, node []byte) ([][]byte, [][]byte, error) { + if bytes.Equal(node, t.emptyHash) { + return [][]byte{}, [][]byte{}, nil + } + // Fetch the node value from the database + nodeValue, err := rTx.Get(node) + if err != nil { + return nil, nil, fmt.Errorf("could not get value for key %x: %w", node, err) + } + + // Depending on the node type, decide what to do + var keys, values [][]byte + switch nodeValue[0] { + case PrefixValueEmpty: + // No leaves in an empty node + + case PrefixValueLeaf: + // Add the leaf value to the list + key, value := ReadLeafValue(nodeValue) + keys = append(keys, key) + values = append(values, value) + + case PrefixValueIntermediate: + if len(nodeValue) != PrefixValueLen+t.hashFunction.Len()*2 { + return nil, nil, fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)", + PrefixValueLen+t.hashFunction.Len()*2, len(nodeValue)) + } + // If it's an intermediate node, traverse its children + lChild, rChild := ReadIntermediateChilds(nodeValue) + + // Fetch leaves from the left child + leftKeys, leftValues, err := t.getLeavesFromSubPath(rTx, lChild) + if err != nil { + return nil, nil, err + } + + // Fetch leaves from the right child + rightKeys, rightValues, err := t.getLeavesFromSubPath(rTx, rChild) + if err != nil { + return nil, nil, err + } + + // Merge leaves from both children + keys = append(keys, leftKeys...) + keys = append(keys, rightKeys...) + values = append(values, leftValues...) + values = append(values, rightValues...) + + default: + return nil, nil, ErrInvalidValuePrefix + } + return keys, values, nil +} diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 7df8850fb..b7327a065 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -438,9 +438,18 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { } // if the neighbor is not empty, set the leaf key to the empty hash + + var neighbourKeys, neighbourValues [][]byte if !bytes.Equal(siblings[len(siblings)-1], t.emptyHash) { - if err := wTx.Set(leafKey, t.emptyHash); err != nil { - return fmt.Errorf("error setting leaf key %x to empty hash: %w", leafKey, err) + neighbourKeys, neighbourValues, err = t.getLeavesFromSubPath(wTx, siblings[len(siblings)-1]) + if err != nil { + return fmt.Errorf("error getting leafs from subpath on Delete: %w", err) + } + // if there are no neighbours, set the leaf key to the empty hash + if len(neighbourKeys) == 0 { + if err := wTx.Set(leafKey, t.emptyHash); err != nil { + return fmt.Errorf("error setting leaf key %x to empty hash: %w", leafKey, err) + } } } else { // else delete the leaf key @@ -464,10 +473,21 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return err } + // Delete the orphan intermediate nodes. if err := deleteNodes(wTx, intermediates); err != nil { return fmt.Errorf("error deleting orphan intermediate nodes: %v", err) } + // If there are neighbours deleted, add them back to the tree. + for i, k := range neighbourKeys { + if err := t.deleteWithTx(wTx, k); err != nil { + return fmt.Errorf("error deleting neighbour %d: %w", i, err) + } + if err := t.addWithTx(wTx, k, neighbourValues[i]); err != nil { + return fmt.Errorf("error adding neighbour %d: %w", i, err) + } + } + // Update the number of leaves. return t.decNLeafs(wTx, 1) } @@ -567,21 +587,9 @@ func (t *Tree) SetRootWithTx(wTx db.WriteTx, root []byte) error { if !t.editable() { return ErrSnapshotNotEditable } - if root == nil { return fmt.Errorf("can not SetRoot with nil root") } - - // check that the root exists in the db - if !bytes.Equal(root, t.emptyHash) { - if _, err := wTx.Get(root); err == ErrKeyNotFound { - return fmt.Errorf("can not SetRoot with root %x, as it"+ - " does not exist in the db", root) - } else if err != nil { - return err - } - } - return wTx.Set(dbKeyRoot, root) } diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 8daa79126..f6c9bbb40 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -1,6 +1,7 @@ package arbo import ( + "bytes" "encoding/hex" "fmt" "math" @@ -664,9 +665,10 @@ func TestSetRoot(t *testing.T) { keys = append(keys, k) values = append(values, v) } - indexes, err := tree.AddBatch(keys, values) - c.Assert(err, qt.IsNil) - c.Check(len(indexes), qt.Equals, 0) + for i, k := range keys { + err := tree.Add(k, values[i]) + c.Assert(err, qt.IsNil) + } checkRootBIString(c, tree, expectedRoot) @@ -688,9 +690,9 @@ func TestSetRoot(t *testing.T) { checkRootBIString(c, tree, expectedRoot) // check that the tree can be updated - err = tree.Add([]byte("test"), []byte("test")) + err = tree.Add(BigIntToBytes(bLen, big.NewInt(int64(1024))), []byte("test")) c.Assert(err, qt.IsNil) - err = tree.Update([]byte("test"), []byte("test")) + err = tree.Update(BigIntToBytes(bLen, big.NewInt(int64(1024))), []byte("test2")) c.Assert(err, qt.IsNil) // check that the k-v '1000' does not exist in the new tree @@ -1112,3 +1114,142 @@ func dirSize(path string) (int64, error) { }) return size, err } + +func TestGetLeavesFromSubPath(t *testing.T) { + c := qt.New(t) + database := metadb.NewTest(t) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, + HashFunction: HashFunctionPoseidon}) + c.Assert(err, qt.IsNil) + + bLen := 32 + // Add keys with shared prefix and some without shared prefix + keys := [][]byte{ + // Shared prefix keys + append([]byte{0xFF, 0xFF}, BigIntToBytes(bLen-2, big.NewInt(int64(1)))...), + append([]byte{0xFF, 0xFF}, BigIntToBytes(bLen-2, big.NewInt(int64(2)))...), + append([]byte{0xFF, 0xFF}, BigIntToBytes(bLen-2, big.NewInt(int64(3)))...), + // Non-shared prefix keys + BigIntToBytes(bLen, big.NewInt(int64(4))), + BigIntToBytes(bLen, big.NewInt(int64(5))), + } + sharedPrefixKeyNotAdded := append([]byte{0xFF, 0xFF}, BigIntToBytes(bLen-2, big.NewInt(int64(4)))...) + values := [][]byte{ + BigIntToBytes(bLen, big.NewInt(int64(10))), + BigIntToBytes(bLen, big.NewInt(int64(20))), + BigIntToBytes(bLen, big.NewInt(int64(30))), + BigIntToBytes(bLen, big.NewInt(int64(40))), + BigIntToBytes(bLen, big.NewInt(int64(50))), + } + for i, key := range keys { + err := tree.Add(key, values[i]) + c.Assert(err, qt.IsNil) + } + + root, err := tree.Root() + c.Assert(err, qt.IsNil) + + // Use the down function to get intermediate nodes on the path to a given leaf with shared prefix + var intermediates [][]byte + _, _, _, err = tree.down( + database, + sharedPrefixKeyNotAdded, + root, + [][]byte{}, + &intermediates, + getPath(tree.maxLevels, sharedPrefixKeyNotAdded), + 0, + false, + ) + c.Assert(err, qt.IsNil) + + // For our test, we'll use the third intermediate node in the path + // (this assumes that it's the one shared by the leaves with the shared prefix) + if len(intermediates) == 0 { + t.Fatal("No intermediates found") + } + intermediateNodeKey := intermediates[2] + + subKeys, _, err := tree.getLeavesFromSubPath(database, intermediateNodeKey) + c.Assert(err, qt.IsNil) + + // Check if all leaves with shared prefix are present in the result and leaves without the shared prefix are not + for _, key := range keys { + found := false + for _, sk := range subKeys { + if bytes.Equal(sk, key) { + found = true + break + } + } + if bytes.HasPrefix(key, []byte{0xFF, 0xFF}) { + c.Assert(found, qt.IsTrue, qt.Commentf("Expected leaf %x not found", key)) + } else { + c.Assert(found, qt.IsFalse, qt.Commentf("Leaf %x should not have been found", key)) + } + } +} + +func TestTreeAfterDeleteAndReconstruct(t *testing.T) { + nLeafs := 1000 + + c := qt.New(t) + database1 := metadb.NewTest(t) + database2 := metadb.NewTest(t) + + // 1. A new tree (tree1) is created and filled with some data + tree1, err := NewTree(Config{Database: database1, MaxLevels: 256, + HashFunction: HashFunctionBlake2b}) + c.Assert(err, qt.IsNil) + + // prepare inputs + var keys, values [][]byte + for i := 0; i < nLeafs; i++ { + k := randomBytes(32) + v := randomBytes(32) + keys = append(keys, k) + values = append(values, v) + } + + for i, key := range keys { + err := tree1.Add(key, values[i]) + c.Assert(err, qt.IsNil) + } + + // 2. Delete some keys from the tree + keysToDelete := [][]byte{keys[1], keys[3]} + for _, key := range keysToDelete { + err := tree1.Delete(key) + c.Assert(err, qt.IsNil) + } + + // 3. Create a second tree (tree2) + tree2, err := NewTree(Config{Database: database2, MaxLevels: 256, + HashFunction: HashFunctionBlake2b}) + c.Assert(err, qt.IsNil) + + // 4. Add the non-deleted keys from tree1 to tree2 + for i, key := range keys { + if !contains(keysToDelete, key) { + err := tree2.Add(key, values[i]) + c.Assert(err, qt.IsNil) + } + } + + root1, err := tree1.Root() + c.Assert(err, qt.IsNil) + root2, err := tree2.Root() + c.Assert(err, qt.IsNil) + // 5. verify the root hash is the same for tree1 and tree2 + c.Assert(bytes.Equal(root1, root2), qt.IsTrue, qt.Commentf("Roots of tree1 and tree2 do not match")) +} + +// Helper function to check if a byte slice array contains a byte slice +func contains(arr [][]byte, item []byte) bool { + for _, a := range arr { + if bytes.Equal(a, item) { + return true + } + } + return false +} From d7807bd22026b279286c1fe163543c3a5acb63be Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 30 Aug 2023 17:18:21 +0200 Subject: [PATCH 10/15] arbo: add RootsFromLevel() method Since we do not hold old intermediate nodes anymore, SetRoot(), Snapshot(), Dump() and so on require to have an existing current intermediate node (or nil to use the current root). To this end we introduce a new method to facilitate retrieve the list of existing roots from a level. --- tree/arbo/navigate.go | 51 ++++++++++++++++++++++++++++++++++++++++++ tree/arbo/tree.go | 29 +++++++++++++++++------- tree/arbo/tree_test.go | 13 ++++++----- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go index c41d4e226..0c5cdc82c 100644 --- a/tree/arbo/navigate.go +++ b/tree/arbo/navigate.go @@ -251,3 +251,54 @@ func (t *Tree) getLeavesFromSubPath(rTx db.Reader, node []byte) ([][]byte, [][]b } return keys, values, nil } + +// RootsFromLevel retrieves all intermediary nodes at a specified level of the tree. +// The function traverses the tree from the root down to the given level, collecting +// all the intermediary nodes along the way. If the specified level exceeds the maximum +// depth of the tree, an error is returned. +func (t *Tree) RootsFromLevel(level int) ([][]byte, error) { + if level > t.maxLevels { + return nil, fmt.Errorf("requested level %d exceeds tree's max level %d", level, t.maxLevels) + } + + var nodes [][]byte + root, err := t.Root() + if err != nil { + return nil, err + } + + err = t.collectNodesAtLevel(t.db, root, 0, level, &nodes) + return nodes, err +} + +func (t *Tree) collectNodesAtLevel(rTx db.Reader, nodeKey []byte, currentLevel, targetLevel int, nodes *[][]byte) error { + nodeValue, err := rTx.Get(nodeKey) + if err != nil { + return fmt.Errorf("could not get value for key %x: %w", nodeKey, err) + } + + // Check the type of the node + switch nodeValue[0] { + case PrefixValueEmpty, PrefixValueLeaf: + // Do nothing for empty nodes and leaves, just return + return nil + case PrefixValueIntermediate: + // If the current level is the target level, add the node to the list + if currentLevel == targetLevel { + *nodes = append(*nodes, nodeKey) + return nil + } + + // Otherwise, continue traversing down the tree + lChild, rChild := ReadIntermediateChilds(nodeValue) + if err := t.collectNodesAtLevel(rTx, lChild, currentLevel+1, targetLevel, nodes); err != nil { + return err + } + if err := t.collectNodesAtLevel(rTx, rChild, currentLevel+1, targetLevel, nodes); err != nil { + return err + } + default: + return ErrInvalidValuePrefix + } + return nil +} diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index b7327a065..45472ca58 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -571,7 +571,9 @@ func (*Tree) GetNLeafsWithTx(rTx db.Reader) (int, error) { return int(nLeafs), nil } -// SetRoot sets the root to the given root +// SetRoot sets the root to the given root. The root hash must be a valid existing +// intermediate node in the tree. The list of roots for a level can be obtained +// using tree.RootsFromLevel(). func (t *Tree) SetRoot(root []byte) error { wTx := t.db.WriteTx() defer wTx.Discard() @@ -593,7 +595,10 @@ func (t *Tree) SetRootWithTx(wTx db.WriteTx, root []byte) error { return wTx.Set(dbKeyRoot, root) } -// Snapshot returns a read-only copy of the Tree from the given root +// Snapshot returns a read-only copy of the Tree from the given root. +// If no root is given, the current root is used. +// The provided root must be a valid existing intermediate node in the tree. +// The list of roots for a level can be obtained using tree.RootsFromLevel(). func (t *Tree) Snapshot(fromRoot []byte) (*Tree, error) { // allow to define which root to use if fromRoot == nil { @@ -626,12 +631,18 @@ func (t *Tree) Snapshot(fromRoot []byte) (*Tree, error) { // Iterate iterates through the full Tree, executing the given function on each // node of the Tree. +// The iteration starts from the given root. If no root is given, the current +// root is used. The provided root must be a valid existing intermediate node in +// the tree. func (t *Tree) Iterate(fromRoot []byte, f func([]byte, []byte)) error { return t.IterateWithTx(t.db, fromRoot, f) } // IterateWithTx does the same than the Iterate method, but allowing to pass // the db.ReadTx that is used. +// The iteration starts from the given root. If no root is given, the current +// root is used. The provided root must be a valid existing intermediate node in +// the tree. func (t *Tree) IterateWithTx(rTx db.Reader, fromRoot []byte, f func([]byte, []byte)) error { // allow to define which root to use if fromRoot == nil { @@ -661,8 +672,7 @@ func (t *Tree) IterateWithStop(fromRoot []byte, f func(int, []byte, []byte) bool // IterateWithStopWithTx does the same than the IterateWithStop method, but // allowing to pass the db.ReadTx that is used. -func (t *Tree) IterateWithStopWithTx(rTx db.Reader, fromRoot []byte, - f func(int, []byte, []byte) bool) error { +func (t *Tree) IterateWithStopWithTx(rTx db.Reader, fromRoot []byte, f func(int, []byte, []byte) bool) error { // allow to define which root to use if fromRoot == nil { var err error @@ -674,8 +684,7 @@ func (t *Tree) IterateWithStopWithTx(rTx db.Reader, fromRoot []byte, return t.iterWithStop(rTx, fromRoot, 0, f) } -func (t *Tree) iterWithStop(rTx db.Reader, k []byte, currLevel int, - f func(int, []byte, []byte) bool) error { +func (t *Tree) iterWithStop(rTx db.Reader, k []byte, currLevel int, f func(int, []byte, []byte) bool) error { var v []byte var err error if bytes.Equal(k, t.emptyHash) { @@ -719,12 +728,16 @@ func (t *Tree) iter(rTx db.Reader, k []byte, f func([]byte, []byte)) error { return t.iterWithStop(rTx, k, 0, f2) } -// Dump exports all the Tree leafs in a byte array +// Dump exports all the Tree leafs in a byte array. +// The provided root must be a valid existing intermediate node in the tree. +// Or nil to use the current root. func (t *Tree) Dump(fromRoot []byte) ([]byte, error) { return t.dump(fromRoot, nil) } -// DumpWriter exports all the Tree leafs writing the bytes in the given Writer +// DumpWriter exports all the Tree leafs writing the bytes in the given Writer. +// The provided root must be a valid existing intermediate node in the tree. +// Or nil to use the current root. func (t *Tree) DumpWriter(fromRoot []byte, w io.Writer) error { _, err := t.dump(fromRoot, w) return err diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index f6c9bbb40..1b755408c 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -680,14 +680,15 @@ func TestSetRoot(t *testing.T) { checkRootBIString(c, tree, "10747149055773881257049574592162159501044114324358186833013814735296193179713") - // do a SetRoot, and expect the same root than the original tree - pastRootBI, ok := new(big.Int).SetString(expectedRoot, 10) - c.Assert(ok, qt.IsTrue) - pastRoot := BigIntToBytes(32, pastRootBI) + // get the roots from level 2 and set one of them as the new root + roots, err := tree.RootsFromLevel(2) + c.Assert(err, qt.IsNil) + c.Assert(tree.SetRoot(roots[0]), qt.IsNil) - err = tree.SetRoot(pastRoot) + // check that the new root is the same as the one set + root, err := tree.Root() c.Assert(err, qt.IsNil) - checkRootBIString(c, tree, expectedRoot) + c.Assert(string(root), qt.Equals, string(roots[0])) // check that the tree can be updated err = tree.Add(BigIntToBytes(bLen, big.NewInt(int64(1024))), []byte("test")) From 1695f3d489ea418c6d322c16acf59fb21f3cfce8 Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 30 Aug 2023 20:05:40 +0200 Subject: [PATCH 11/15] run mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4f8d66c3a..67c724cd3 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 + github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a github.com/vocdoni/go-snark v0.0.0-20210709152824-f6e4c27d7319 github.com/vocdoni/storage-proofs-eth-go v0.1.6 go.vocdoni.io/proto v1.14.6-0.20230802094125-e07a41fda290 @@ -261,7 +262,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/stretchr/testify v1.8.4 // indirect github.com/subosito/gotenv v1.4.2 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect From d2f2e6048d412d87332416077d0a892c4c23fe37 Mon Sep 17 00:00:00 2001 From: p4u Date: Thu, 31 Aug 2023 23:08:47 +0200 Subject: [PATCH 12/15] final fixes after arbo refactor The main issue was that on update, if the key and value are the same, the clean up routine for removing old intermediate nodes, was removing the current valid intermediate nodes up to the root. In addition adapt some tests to become compatible with the new impl. And remove some that are not longer valid. --- dockerfiles/testsuite/env.gateway0 | 2 +- dockerfiles/testsuite/genesis.json | 12 +- statedb/treeupdate.go | 31 +++-- statedb/treeview.go | 2 +- tree/arbo/addbatch_test.go | 174 ----------------------------- tree/arbo/navigate.go | 7 +- tree/arbo/tree.go | 68 +++++++---- tree/arbo/tree_test.go | 54 +-------- tree/tree.go | 7 +- vochain/account_test.go | 28 +++-- 10 files changed, 106 insertions(+), 279 deletions(-) diff --git a/dockerfiles/testsuite/env.gateway0 b/dockerfiles/testsuite/env.gateway0 index 9bb02fd06..7d4d833e9 100755 --- a/dockerfiles/testsuite/env.gateway0 +++ b/dockerfiles/testsuite/env.gateway0 @@ -3,7 +3,7 @@ VOCDONI_LOGLEVEL=debug VOCDONI_DEV=True VOCDONI_ENABLEAPI=True VOCDONI_ENABLERPC=True -VOCDONI_ENABLEFAUCETWITHAMOUNT=1000 +VOCDONI_ENABLEFAUCETWITHAMOUNT=100000 VOCDONI_VOCHAIN_SEEDS=3c3765494e758ae7baccb1f5b0661755302ddc47@seed:26656 VOCDONI_VOCHAIN_GENESIS=/app/misc/genesis.json VOCDONI_VOCHAIN_NOWAITSYNC=True diff --git a/dockerfiles/testsuite/genesis.json b/dockerfiles/testsuite/genesis.json index 7e05792cb..98d395039 100755 --- a/dockerfiles/testsuite/genesis.json +++ b/dockerfiles/testsuite/genesis.json @@ -23,8 +23,8 @@ }, "app_hash": "", "app_state": { - "max_election_size": 100000, - "network_capacity": 4000, + "max_election_size": 1000000, + "network_capacity": 10000, "validators": [ { "signer_address": "6bcc92be71d48cfe3f2f5042f157ad978f3e8117", @@ -58,19 +58,19 @@ "accounts":[ { "address":"0xccEc2c2D658261Fbdc40b04FEc06d49057242D39", - "balance":100000 + "balance":10000000 }, { "address":"0x776d858D17C8018F07899dB535866EBf805a32E0", - "balance":100000 + "balance":10000000 }, { "address":"0x074fcAacb8B01850539eaE7E9fEE8dc94549db96", - "balance":100000 + "balance":10000000 }, { "address":"0x88a499cEf9D1330111b41360173967c9C1bf703f", - "balance":100000 + "balance":10000000 } ], "treasurer": "0xfe10DAB06D636647f4E40dFd56599da9eF66Db1c", diff --git a/statedb/treeupdate.go b/statedb/treeupdate.go index dadaccc02..c19771d5d 100644 --- a/statedb/treeupdate.go +++ b/statedb/treeupdate.go @@ -2,6 +2,7 @@ package statedb import ( "bytes" + "fmt" "io" "path" "sync" @@ -136,6 +137,11 @@ func (u *TreeUpdate) Del(key []byte) error { return u.tree.Del(u.tree.tx, key) } +// PrintGraphviz prints the tree in Graphviz format. +func (u *TreeUpdate) PrintGraphviz() error { + return u.tree.PrintGraphviz(u.tree.tx) +} + // SubTree is used to open the subTree (singleton and non-singleton) as a // TreeUpdate. The treeUpdate.tx is created from u.tx appending the prefix // `subKeySubTree | cfg.prefix`. In turn the treeUpdate.tree uses the @@ -164,9 +170,12 @@ func (u *TreeUpdate) SubTree(cfg TreeConfig) (treeUpdate *TreeUpdate, err error) return nil, err } if !bytes.Equal(root, lastRoot) { - if err := tree.SetRoot(txTree, root); err != nil { - return nil, err - } + panic(fmt.Sprintf("root for %s mismatch: %x != %x", cfg.kindID, root, lastRoot)) + // Note (Pau): since we modified arbo to remove all unecessary intermediate nodes, + // we cannot set a past root.We should probably remove this code. + //if err := tree.SetRoot(txTree, root); err != nil { + // return nil, err + //} } treeUpdate = &TreeUpdate{ tx: tx, @@ -296,16 +305,16 @@ func propagateRoot(treeUpdate *TreeUpdate) ([]byte, error) { for key, updates := range updatesByKey { leaf, err := treeUpdate.tree.Get(treeUpdate.tree.tx, []byte(key)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get leaf %x: %w", key, err) } for _, update := range updates { leaf, err = update.setRoot(leaf, update.root) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to set root for leaf %x: %w", key, err) } } if err := treeUpdate.tree.Set(treeUpdate.tree.tx, []byte(key), leaf); err != nil { - return nil, err + return nil, fmt.Errorf("failed to set leaf %x: %w", key, err) } } // We either updated leaves here, or treeUpdate.tree was dirty, so we @@ -324,22 +333,22 @@ func propagateRoot(treeUpdate *TreeUpdate) ([]byte, error) { func (t *TreeTx) Commit(version uint32) error { root, err := propagateRoot(&t.TreeUpdate) if err != nil { - return err + return fmt.Errorf("could not propagate root: %w", err) } t.openSubs = sync.Map{} curVersion, err := getVersion(t.tx) if err != nil { - return err + return fmt.Errorf("could not get current version: %w", err) } // If root is nil, it means that there were no updates to the StateDB, // so the next version root is the current version root. if root == nil { if root, err = t.sdb.getVersionRoot(t.tx, curVersion); err != nil { - return err + return fmt.Errorf("could not get current version root: %w", err) } } if err := setVersionRoot(t.tx, version, root); err != nil { - return err + return fmt.Errorf("could not set version root: %w", err) } return t.tx.Commit() } @@ -407,7 +416,7 @@ func (v *treeUpdateView) Import(r io.Reader) error { } func (v *treeUpdateView) PrintGraphviz() error { - return v.tree.PrintGraphviz() + return v.tree.PrintGraphviz(v.tx) } // verify that treeUpdateView fulfills the TreeViewer interface. diff --git a/statedb/treeview.go b/statedb/treeview.go index c0a735ef4..f96991869 100644 --- a/statedb/treeview.go +++ b/statedb/treeview.go @@ -83,7 +83,7 @@ func (v *TreeView) IterateNodes(callback func(key, value []byte) bool) error { } func (v *TreeView) PrintGraphviz() error { - return v.tree.PrintGraphviz() + return v.tree.PrintGraphviz(v.db) } // Iterate iterates over all leafs of this tree. When callback returns true, diff --git a/tree/arbo/addbatch_test.go b/tree/arbo/addbatch_test.go index 6c77694e8..e662191d5 100644 --- a/tree/arbo/addbatch_test.go +++ b/tree/arbo/addbatch_test.go @@ -1,7 +1,6 @@ package arbo import ( - "bytes" "crypto/rand" "encoding/hex" "fmt" @@ -975,176 +974,3 @@ func TestAddKeysWithEmptyValues(t *testing.T) { c.Assert(err, qt.IsNil) c.Check(verif, qt.IsFalse) } - -/* -func TestAddBatchThresholdInDisk(t *testing.T) { - c := qt.New(t) - - // customize thresholdNLeafs for the test - testThresholdNLeafs := 1024 - - database1 := metadb.NewTest(t) - tree1, err := NewTree(Config{database1, 256, testThresholdNLeafs, - HashFunctionBlake2b}) - c.Assert(err, qt.IsNil) - - database2, err := pebbledb.New(db.Options{Path: c.TempDir()}) - c.Assert(err, qt.IsNil) - tree2, err := NewTree(Config{database2, 256, testThresholdNLeafs, - HashFunctionBlake2b}) - c.Assert(err, qt.IsNil) - - database3, err := pebbledb.New(db.Options{Path: c.TempDir()}) - c.Assert(err, qt.IsNil) - tree3, err := NewTree(Config{database3, 256, testThresholdNLeafs, - HashFunctionBlake2b}) - c.Assert(err, qt.IsNil) - - var keys, values [][]byte - for i := 0; i < 3*testThresholdNLeafs; i++ { - k := randomBytes(32) - v := randomBytes(32) - if err := tree1.Add(k, v); err != nil { - t.Fatal(err) - } - if i < testThresholdNLeafs+1 { - if err := tree2.Add(k, v); err != nil { - t.Fatal(err) - } - } - // store for later addition through AddBatch - keys = append(keys, k) - values = append(values, v) - } - - invalids, err := tree2.AddBatch(keys[testThresholdNLeafs+1:], values[testThresholdNLeafs+1:]) - c.Assert(err, qt.IsNil) - c.Check(len(invalids), qt.Equals, 0) - // check that both trees roots are equal - checkRoots(c, tree1, tree2) - - // call directly the tree3.addBatchInDisk to ensure that is tested - wTx := tree3.db.WriteTx() - defer wTx.Discard() - invalids, err = tree3.addBatchInDisk(wTx, keys, values) - c.Assert(err, qt.IsNil) - err = wTx.Commit() - c.Assert(err, qt.IsNil) - c.Check(len(invalids), qt.Equals, 0) - // check that both trees roots are equal - checkRoots(c, tree1, tree3) - - // now add one leaf more to the trees to ensure that the previous - // actions did not left the tree in an invalid state - k := randomBytes(32) - v := randomBytes(32) - err = tree1.Add(k, v) - c.Assert(err, qt.IsNil) - err = tree2.Add(k, v) - c.Assert(err, qt.IsNil) - err = tree3.Add(k, v) - c.Assert(err, qt.IsNil) -} -*/ - -func initTestUpFromSubRoots(c *qt.C) (*Tree, *Tree) { - database1 := metadb.NewTest(c) - tree1, err := NewTree(Config{database1, 256, DefaultThresholdNLeafs, - HashFunctionBlake2b}) - c.Assert(err, qt.IsNil) - - database2 := metadb.NewTest(c) - tree2, err := NewTree(Config{database2, 256, DefaultThresholdNLeafs, - HashFunctionBlake2b}) - c.Assert(err, qt.IsNil) - return tree1, tree2 -} - -func testUpFromSubRoots(c *qt.C, tree1, tree2 *Tree, preSubRoots [][]byte) { - // add the preSubRoots to the tree1 - for i := 0; i < len(preSubRoots); i++ { - if bytes.Equal(preSubRoots[i], tree1.emptyHash) { - continue - } - err := tree1.Add(preSubRoots[i], nil) - c.Assert(err, qt.IsNil) - } - root1, err := tree1.Root() - c.Assert(err, qt.IsNil) - - wTx := tree2.db.WriteTx() - subRoots := make([][]byte, len(preSubRoots)) - for i := 0; i < len(preSubRoots); i++ { - if preSubRoots[i] == nil || bytes.Equal(preSubRoots[i], tree1.emptyHash) { - subRoots[i] = tree1.emptyHash - continue - } - leafKey, leafValue, err := tree2.newLeafValue(preSubRoots[i], nil) - c.Assert(err, qt.IsNil) - subRoots[i] = leafKey - - err = wTx.Set(leafKey, leafValue) - c.Assert(err, qt.IsNil) - } - // first fill the leaf nodes - // then call upFromSubRoots - root2FromUp, err := tree2.upFromSubRoots(wTx, subRoots) - c.Assert(err, qt.IsNil) - - err = tree2.SetRootWithTx(wTx, root2FromUp) - c.Assert(err, qt.IsNil) - err = wTx.Commit() - c.Assert(err, qt.IsNil) - - root2, err := tree2.Root() - c.Assert(err, qt.IsNil) - - c.Assert(root1, qt.DeepEquals, root2) -} - -func testUpFromSubRootsWithEmpties(c *qt.C, preSubRoots [][]byte, indexEmpties []int) { - tree1, tree2 := initTestUpFromSubRoots(c) - - testPreSubRoots := make([][]byte, len(preSubRoots)) - copy(testPreSubRoots, preSubRoots) - for i := 0; i < len(indexEmpties); i++ { - testPreSubRoots[indexEmpties[i]] = tree1.emptyHash - } - testUpFromSubRoots(c, tree1, tree2, testPreSubRoots) -} - -func TestUpFromSubRoots(t *testing.T) { - c := qt.New(t) - - // prepare preSubRoots - preSubRoots := [][]byte{ - BigIntToBytes(32, big.NewInt(4)), - BigIntToBytes(32, big.NewInt(2)), - BigIntToBytes(32, big.NewInt(1)), - BigIntToBytes(32, big.NewInt(3)), - } - - // test using the full 4 leafs as subRoots - testUpFromSubRootsWithEmpties(c, preSubRoots, nil) - // 1st subRoot empty - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{0}) - // 2nd subRoot empty - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{1}) - // 3rd subRoot empty - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{2}) - // 4th subRoot empty - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{3}) - - // other combinations of empty SubRoots - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{0, 1, 2, 3}) - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{0, 1}) - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{1, 2}) - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{1, 3}) - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{2, 3}) - testUpFromSubRootsWithEmpties(c, preSubRoots, []int{0, 2, 3}) -} - -// TODO test adding batch with multiple invalid keys -// TODO for tests of AddBatch, if the root does not match the Add root, bulk -// all the leafs of both trees into a log file to later be able to debug and -// recreate the case diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go index 0c5cdc82c..2ce83581d 100644 --- a/tree/arbo/navigate.go +++ b/tree/arbo/navigate.go @@ -11,6 +11,7 @@ import ( // down goes down to the leaf recursively func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, intermediates *[][]byte, path []bool, currLvl int, getLeaf bool) ([]byte, []byte, [][]byte, error) { + if currLvl > t.maxLevels { return nil, nil, nil, ErrMaxLevel } @@ -23,7 +24,7 @@ func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, in } currValue, err = rTx.Get(currKey) if err != nil { - return nil, nil, nil, fmt.Errorf("could not get value for key %x: %w", currKey, err) + return nil, nil, nil, fmt.Errorf("while down, could not get value for key %x: %w", currKey, err) } switch currValue[0] { @@ -65,7 +66,9 @@ func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, in fmt.Errorf("intermediate value invalid length (expected: %d, actual: %d)", PrefixValueLen+t.hashFunction.Len()*2, len(currValue)) } - *intermediates = append(*intermediates, currKey) + if intermediates != nil { + *intermediates = append(*intermediates, currKey) + } // collect siblings while going down if path[currLvl] { diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 45472ca58..5b32ce5a1 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -357,24 +357,31 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { return ErrKeyNotFound } - // delete the old leaf key - if err := wTx.Delete(oldLeafKey); err != nil { - return fmt.Errorf("error deleting old leaf on update: %w", err) - } - // compute the new leaf key and value leafKey, leafValue, err := t.newLeafValue(k, v) if err != nil { return err } + if bytes.Equal(oldLeafKey, leafKey) { + // The key is the same, just return, nothing to do. + // Note that if we don't exit here, the code below will delete + // the valid intermediate nodes up to the root, and we don't want that. + // (took me a while to figure out) + return nil + } + + // delete the old leaf key + if err := wTx.Delete(oldLeafKey); err != nil { + return fmt.Errorf("error deleting old leaf on update: %w", err) + } + // add the new leaf key if err := wTx.Set(leafKey, leafValue); err != nil { return err } // go up to the root - if len(siblings) == 0 { return t.setRoot(wTx, leafKey) } @@ -384,6 +391,7 @@ func (t *Tree) UpdateWithTx(wTx db.WriteTx, k, v []byte) error { return err } + // delete the old intermediate nodes if err := deleteNodes(wTx, intermediates); err != nil { return fmt.Errorf("error deleting orphan intermediate nodes: %v", err) } @@ -438,6 +446,9 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { } // if the neighbor is not empty, set the leaf key to the empty hash + if len(siblings) == 0 { + return nil + } var neighbourKeys, neighbourValues [][]byte if !bytes.Equal(siblings[len(siblings)-1], t.emptyHash) { @@ -517,9 +528,9 @@ func (t *Tree) GetWithTx(rTx db.Reader, k []byte) ([]byte, []byte, error) { } // go down to the leaf - var siblings, intermediates [][]byte + var siblings [][]byte - _, value, _, err := t.down(rTx, k, root, siblings, &intermediates, path, 0, true) + _, value, _, err := t.down(rTx, k, root, siblings, nil, path, 0, true) if err != nil { return nil, nil, err } @@ -592,6 +603,10 @@ func (t *Tree) SetRootWithTx(wTx db.WriteTx, root []byte) error { if root == nil { return fmt.Errorf("can not SetRoot with nil root") } + if _, err := wTx.Get(root); err != nil { + return fmt.Errorf("could not SetRoot to %x,"+ + " as it does not exist in the db: %w", root, err) + } return wTx.Set(dbKeyRoot, root) } @@ -862,13 +877,13 @@ func (t *Tree) ImportDumpReader(r io.Reader) error { // Graphviz iterates across the full tree to generate a string Graphviz // representation of the tree and writes it to w func (t *Tree) Graphviz(w io.Writer, fromRoot []byte) error { - return t.GraphvizFirstNLevels(w, fromRoot, t.maxLevels) + return t.GraphvizFirstNLevels(t.db, w, fromRoot, t.maxLevels) } // GraphvizFirstNLevels iterates across the first NLevels of the tree to // generate a string Graphviz representation of the first NLevels of the tree // and writes it to w -func (t *Tree) GraphvizFirstNLevels(w io.Writer, fromRoot []byte, untilLvl int) error { +func (t *Tree) GraphvizFirstNLevels(rTx db.Reader, w io.Writer, fromRoot []byte, untilLvl int) error { fmt.Fprintf(w, `digraph hierarchy { node [fontname=Monospace,fontsize=10,shape=box] `) @@ -882,25 +897,31 @@ node [fontname=Monospace,fontsize=10,shape=box] } nEmpties := 0 - err := t.iterWithStop(t.db, fromRoot, 0, func(currLvl int, k, v []byte) bool { + err := t.iterWithStop(rTx, fromRoot, 0, func(currLvl int, k, v []byte) bool { if currLvl == untilLvl { return true // to stop the iter from going down } + firstChars := func(b []byte) string { + if len(b) > nChars { + return hex.EncodeToString(b[:nChars]) + } + return hex.EncodeToString(b) + } switch v[0] { case PrefixValueEmpty: case PrefixValueLeaf: - fmt.Fprintf(w, "\"%v\" [style=filled];\n", hex.EncodeToString(k[:nChars])) + fmt.Fprintf(w, "\"%v\" [style=filled];\n", firstChars(k)) // key & value from the leaf kB, vB := ReadLeafValue(v) fmt.Fprintf(w, "\"%v\" -> {\"k:%v\\nv:%v\"}\n", - hex.EncodeToString(k[:nChars]), hex.EncodeToString(kB[:nChars]), - hex.EncodeToString(vB[:nChars])) + firstChars(k), firstChars(kB), + firstChars(vB)) fmt.Fprintf(w, "\"k:%v\\nv:%v\" [style=dashed]\n", - hex.EncodeToString(kB[:nChars]), hex.EncodeToString(vB[:nChars])) + firstChars(kB), firstChars(vB)) case PrefixValueIntermediate: l, r := ReadIntermediateChilds(v) - lStr := hex.EncodeToString(l[:nChars]) - rStr := hex.EncodeToString(r[:nChars]) + lStr := firstChars(l) + rStr := firstChars(r) eStr := "" if bytes.Equal(l, t.emptyHash) { lStr = fmt.Sprintf("empty%v", nEmpties) @@ -914,7 +935,7 @@ node [fontname=Monospace,fontsize=10,shape=box] rStr) nEmpties++ } - fmt.Fprintf(w, "\"%v\" -> {\"%v\" \"%v\"}\n", hex.EncodeToString(k[:nChars]), + fmt.Fprintf(w, "\"%v\" -> {\"%v\" \"%v\"}\n", firstChars(k), lStr, rStr) fmt.Fprint(w, eStr) default: @@ -934,22 +955,25 @@ func (t *Tree) PrintGraphviz(fromRoot []byte) error { return err } } - return t.PrintGraphvizFirstNLevels(fromRoot, t.maxLevels) + return t.PrintGraphvizFirstNLevels(t.db, fromRoot, t.maxLevels) } // PrintGraphvizFirstNLevels prints the output of Tree.GraphvizFirstNLevels -func (t *Tree) PrintGraphvizFirstNLevels(fromRoot []byte, untilLvl int) error { +func (t *Tree) PrintGraphvizFirstNLevels(rTx db.Reader, fromRoot []byte, untilLvl int) error { if fromRoot == nil { var err error - fromRoot, err = t.Root() + fromRoot, err = t.RootWithTx(rTx) if err != nil { return err } } + if untilLvl == 0 { + untilLvl = t.maxLevels + } w := bytes.NewBufferString("") fmt.Fprintf(w, "--------\nGraphviz of the Tree with Root "+hex.EncodeToString(fromRoot)+":\n") - err := t.GraphvizFirstNLevels(w, fromRoot, untilLvl) + err := t.GraphvizFirstNLevels(rTx, w, fromRoot, untilLvl) if err != nil { fmt.Println(w) return err diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 1b755408c..b68a77025 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -537,56 +537,6 @@ func TestRWMutex(t *testing.T) { } } -// TODO UPDATE -// func TestSetGetNLeafs(t *testing.T) { -// c := qt.New(t) -// database := metadb.NewTest(t).IsNil) -// tree, err := NewTree(database, 100, HashFunctionPoseidon) -// c.Assert(err, qt.IsNil) -// -// // 0 -// tree.dbBatch = tree.db.NewBatch() -// -// err = tree.setNLeafs(0) -// c.Assert(err, qt.IsNil) -// -// err = tree.dbBatch.Write() -// c.Assert(err, qt.IsNil) -// -// n, err := tree.GetNLeafs() -// c.Assert(err, qt.IsNil) -// c.Assert(n, qt.Equals, 0) -// -// // 1024 -// tree.dbBatch = tree.db.NewBatch() -// -// err = tree.setNLeafs(1024) -// c.Assert(err, qt.IsNil) -// -// err = tree.dbBatch.Write() -// c.Assert(err, qt.IsNil) -// -// n, err = tree.GetNLeafs() -// c.Assert(err, qt.IsNil) -// c.Assert(n, qt.Equals, 1024) -// -// // 2**64 -1 -// tree.dbBatch = tree.db.NewBatch() -// -// maxUint := ^uint(0) -// maxInt := int(maxUint >> 1) -// -// err = tree.setNLeafs(maxInt) -// c.Assert(err, qt.IsNil) -// -// err = tree.dbBatch.Write() -// c.Assert(err, qt.IsNil) -// -// n, err = tree.GetNLeafs() -// c.Assert(err, qt.IsNil) -// c.Assert(n, qt.Equals, maxInt) -// } - func TestAddBatchFullyUsed(t *testing.T) { c := qt.New(t) @@ -700,9 +650,9 @@ func TestSetRoot(t *testing.T) { _, _, err = tree.Get(k) c.Assert(err, qt.Equals, ErrKeyNotFound) - // check that can be set an empty root + // check that the root can't be set to an empty root err = tree.SetRoot(tree.emptyHash) - c.Assert(err, qt.IsNil) + c.Assert(err, qt.IsNotNil) } func TestSnapshot(t *testing.T) { diff --git a/tree/tree.go b/tree/tree.go index 79e5aacb1..880f2e070 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -285,6 +285,9 @@ func (t *Tree) ImportDump(b []byte) error { return t.tree.ImportDump(b) } -func (t *Tree) PrintGraphviz() error { - return t.tree.PrintGraphviz(nil) +func (t *Tree) PrintGraphviz(rTx db.Reader) error { + if rTx == nil { + rTx = t.db + } + return t.tree.PrintGraphvizFirstNLevels(rTx, nil, 0) } diff --git a/vochain/account_test.go b/vochain/account_test.go index a2ff6a96c..5146966a7 100644 --- a/vochain/account_test.go +++ b/vochain/account_test.go @@ -38,16 +38,28 @@ func setupTestBaseApplicationAndSigners(t *testing.T, signers[i] = &signer } // create burn account - app.State.SetAccount(state.BurnAddress, &state.Account{}) + if err := app.State.SetAccount(state.BurnAddress, &state.Account{}); err != nil { + return nil, nil, err + } // set tx costs - app.State.SetTxBaseCost(models.TxType_SET_ACCOUNT_INFO_URI, 100) - app.State.SetTxBaseCost(models.TxType_ADD_DELEGATE_FOR_ACCOUNT, 100) - app.State.SetTxBaseCost(models.TxType_DEL_DELEGATE_FOR_ACCOUNT, 100) - app.State.SetTxBaseCost(models.TxType_CREATE_ACCOUNT, 100) - app.State.SetTxBaseCost(models.TxType_COLLECT_FAUCET, 100) + if err := app.State.SetTxBaseCost(models.TxType_SET_ACCOUNT_INFO_URI, 100); err != nil { + return nil, nil, err + } + if err := app.State.SetTxBaseCost(models.TxType_ADD_DELEGATE_FOR_ACCOUNT, 100); err != nil { + return nil, nil, err + } + if err := app.State.SetTxBaseCost(models.TxType_DEL_DELEGATE_FOR_ACCOUNT, 100); err != nil { + return nil, nil, err + } + if err := app.State.SetTxBaseCost(models.TxType_CREATE_ACCOUNT, 100); err != nil { + return nil, nil, err + } + if err := app.State.SetTxBaseCost(models.TxType_COLLECT_FAUCET, 100); err != nil { + return nil, nil, err + } // save state - app.Commit(context.TODO(), nil) - return app, signers, nil + _, err := app.Commit(context.TODO(), nil) + return app, signers, err } func TestSetAccountTx(t *testing.T) { From bb970dd93349d199080fbded35bff03e6be5dabc Mon Sep 17 00:00:00 2001 From: p4u Date: Fri, 1 Sep 2023 11:24:45 +0200 Subject: [PATCH 13/15] arbo: handle the case when delete when only 1 element in the tree --- cmd/cli/vocdonicli.go | 3 ++- tree/arbo/navigate.go | 8 ++++---- tree/arbo/tree.go | 35 ++++++++++++++++++++--------------- tree/arbo/tree_test.go | 37 ++++++++++++++++++++++++++++++++++++- vochain/state/sik_test.go | 5 +++-- 5 files changed, 65 insertions(+), 23 deletions(-) diff --git a/cmd/cli/vocdonicli.go b/cmd/cli/vocdonicli.go index ac4bfb278..9141d5b70 100644 --- a/cmd/cli/vocdonicli.go +++ b/cmd/cli/vocdonicli.go @@ -31,6 +31,7 @@ func (c *Config) Load(filepath string) error { data, err := os.ReadFile(filepath) if err != nil { if errors.Is(err, os.ErrNotExist) { + c.LastAccountUsed = -1 return nil } return err @@ -94,7 +95,7 @@ func NewVocdoniCLI(configFile, host string) (*vocdoniCLI, error) { if err != nil { return nil, err } - if len(cfg.Accounts)-1 >= cfg.LastAccountUsed { + if len(cfg.Accounts)-1 >= cfg.LastAccountUsed && cfg.LastAccountUsed >= 0 { log.Infof("using account %d", cfg.LastAccountUsed) if err := api.SetAccount(cfg.Accounts[cfg.LastAccountUsed].PrivKey.String()); err != nil { return nil, err diff --git a/tree/arbo/navigate.go b/tree/arbo/navigate.go index 2ce83581d..53e596a6c 100644 --- a/tree/arbo/navigate.go +++ b/tree/arbo/navigate.go @@ -11,7 +11,6 @@ import ( // down goes down to the leaf recursively func (t *Tree) down(rTx db.Reader, newKey, currKey []byte, siblings [][]byte, intermediates *[][]byte, path []bool, currLvl int, getLeaf bool) ([]byte, []byte, [][]byte, error) { - if currLvl > t.maxLevels { return nil, nil, nil, ErrMaxLevel } @@ -184,10 +183,11 @@ func ReadIntermediateChilds(b []byte) ([]byte, []byte) { func getPath(numLevels int, k []byte) []bool { requiredLen := (numLevels + 7) / 8 // Calculate the ceil value of numLevels/8 + + // If the provided key is shorter than expected, extend it with zero bytes if len(k) < requiredLen { - // The provided key is shorter than expected for the given number of levels. - // Handle this case appropriately, either by returning an error or by handling it in some other way. - panic(fmt.Sprintf("key length is too short: expected at least %d bytes, got %d bytes", requiredLen, len(k))) + padding := make([]byte, requiredLen-len(k)) + k = append(k, padding...) } path := make([]bool, numLevels) diff --git a/tree/arbo/tree.go b/tree/arbo/tree.go index 5b32ce5a1..eb174e356 100644 --- a/tree/arbo/tree.go +++ b/tree/arbo/tree.go @@ -157,13 +157,7 @@ func NewTreeWithTx(wTx db.WriteTx, cfg Config) (*Tree, error) { _, err = wTx.Get(dbKeyRoot) if err == db.ErrKeyNotFound { // store new root 0 (empty) - if err = wTx.Set(dbKeyRoot, t.emptyHash); err != nil { - return nil, err - } - if err = t.setNLeafs(wTx, 0); err != nil { - return nil, err - } - return &t, nil + return &t, t.setToEmptyTree(wTx) } else if err != nil { return nil, fmt.Errorf("unknown database error: %w", err) } @@ -286,9 +280,11 @@ func (t *Tree) add(wTx db.WriteTx, root []byte, fromLvl int, k, v []byte) ([]byt // go up to the root if len(siblings) == 0 { - // return the leafKey as root + // if there are no siblings, means that the tree is empty. + // We consider the leafKey as root return leafKey, nil } + root, err = t.up(wTx, leafKey, siblings, path, len(siblings)-1, fromLvl) if err != nil { return nil, err @@ -425,6 +421,14 @@ func (t *Tree) DeleteWithTx(wTx db.WriteTx, k []byte) error { return t.deleteWithTx(wTx, k) } +// setToEmptyTree sets the tree to an empty tree. +func (t *Tree) setToEmptyTree(wTx db.WriteTx) error { + if err := wTx.Set(dbKeyRoot, t.emptyHash); err != nil { + return err + } + return t.setNLeafs(wTx, 0) +} + // deleteWithTx deletes a key-value pair from the tree. func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { root, err := t.RootWithTx(wTx) @@ -445,12 +449,17 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return err } - // if the neighbor is not empty, set the leaf key to the empty hash if len(siblings) == 0 { - return nil + // if no siblings, means the tree is now empty, + // just delete the leaf key and set the root to empty + if err := wTx.Delete(leafKey); err != nil { + return fmt.Errorf("error deleting leaf key %x: %w", leafKey, err) + } + return t.setToEmptyTree(wTx) } var neighbourKeys, neighbourValues [][]byte + // if the neighbour is not empty, we set it to empty (instead of remove) if !bytes.Equal(siblings[len(siblings)-1], t.emptyHash) { neighbourKeys, neighbourValues, err = t.getLeavesFromSubPath(wTx, siblings[len(siblings)-1]) if err != nil { @@ -470,10 +479,6 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { } // Navigate back up the tree, updating the intermediate nodes. - if len(siblings) == 0 { - // The tree is empty after deletion. - return t.setRoot(wTx, t.emptyHash) - } newRoot, err := t.up(wTx, t.emptyHash, siblings, path, len(siblings)-1, 0) if err != nil { return err @@ -489,7 +494,7 @@ func (t *Tree) deleteWithTx(wTx db.WriteTx, k []byte) error { return fmt.Errorf("error deleting orphan intermediate nodes: %v", err) } - // If there are neighbours deleted, add them back to the tree. + // Delete the neighbour's childs and add them back to the tree in the right place. for i, k := range neighbourKeys { if err := t.deleteWithTx(wTx, k); err != nil { return fmt.Errorf("error deleting neighbour %d: %w", i, err) diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index b68a77025..12ef9fa13 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -947,7 +947,7 @@ func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) { func TestDiskSizeBench(t *testing.T) { c := qt.New(t) - nLeafs := 2000 + nLeafs := 500 printTestContext("TestDiskSizeBench: ", nLeafs, "Blake2b", "pebble") // prepare inputs @@ -1204,3 +1204,38 @@ func contains(arr [][]byte, item []byte) bool { } return false } + +func TestTreeWithSingleLeaf(t *testing.T) { + c := qt.New(t) + + database := metadb.NewTest(t) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, + HashFunction: HashFunctionPoseidon}) + c.Assert(err, qt.IsNil) + + // check empty root + root, err := tree.Root() + c.Assert(err, qt.IsNil) + c.Assert(hex.EncodeToString(root), qt.DeepEquals, "0000000000000000000000000000000000000000000000000000000000000000") + + // add one entry + err = tree.Add([]byte{0x01}, []byte{0x01}) + c.Assert(err, qt.IsNil) + + root, err = tree.Root() + c.Assert(err, qt.IsNil) + c.Assert(root, qt.HasLen, 32) + + leafKey, _, err := newLeafValue(HashFunctionPoseidon, []byte{0x01}, []byte{0x01}) + c.Assert(err, qt.IsNil) + + // check leaf key is actually the current root + c.Assert(bytes.Equal(root, leafKey), qt.IsTrue) + + // delete the only entry and check the tree is empty again + err = tree.Delete([]byte{0x01}) + c.Assert(err, qt.IsNil) + root, err = tree.Root() + c.Assert(err, qt.IsNil) + c.Assert(hex.EncodeToString(root), qt.DeepEquals, "0000000000000000000000000000000000000000000000000000000000000000") +} diff --git a/vochain/state/sik_test.go b/vochain/state/sik_test.go index 37ccd2fd0..a2531c6e7 100644 --- a/vochain/state/sik_test.go +++ b/vochain/state/sik_test.go @@ -159,13 +159,14 @@ func TestAssignSIKToElectionAndPurge(t *testing.T) { c.Assert(err, qt.IsNil) _, err = s.NoState(true).Get(key) c.Assert(err, qt.IsNil) + // purge siks and check c.Assert(s.PurgeSIKsByElection(pid), qt.IsNil) c.Assert(err, qt.IsNil) _, err = s.NoState(true).Get(key) c.Assert(err, qt.IsNotNil) - _, err = s.SIKFromAddress(testAccount.Address()) - c.Assert(err, qt.IsNotNil) + sik, err = s.SIKFromAddress(testAccount.Address()) + c.Assert(err, qt.IsNotNil, qt.Commentf("SIK should be deleted")) } func Test_heightEncoding(t *testing.T) { From 960d9f31dcb214b34b345ea4830e71283264fdd2 Mon Sep 17 00:00:00 2001 From: p4u Date: Fri, 1 Sep 2023 12:55:16 +0200 Subject: [PATCH 14/15] events: notify listeners before commit state tx --- vochain/indexer/indexer.go | 8 +++++++- vochain/state/state.go | 29 +++++++++++++++++------------ vochain/state/votes.go | 1 + 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 17599144d..27c02c66c 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -363,7 +363,13 @@ func (idx *Indexer) Commit(height uint32) error { // TODO: can we get previousVote from sqlite via blockTx? var previousVote *models.StateDBVote if v.Overwrites > 0 { - previousVote, _ = idx.App.State.Vote(v.ProcessID, v.Nullifier, true) + previousVote, err = idx.App.State.Vote(v.ProcessID, v.Nullifier, true) + if err != nil { + log.Warnw("cannot get previous vote", + "nullifier", hex.EncodeToString(v.Nullifier), + "processID", hex.EncodeToString(v.ProcessID), + "error", err.Error()) + } } if previousVote != nil { log.Debugw("vote overwrite, previous vote", diff --git a/vochain/state/state.go b/vochain/state/state.go index f3d3a27da..760f3805a 100644 --- a/vochain/state/state.go +++ b/vochain/state/state.go @@ -386,6 +386,23 @@ func (v *State) RevealProcessKeys(tx *models.AdminTx) error { func (v *State) Save() ([]byte, error) { height := v.CurrentHeight() var pidsStartNextBlock [][]byte + + // Notify listeners about processes that start in the next block. + if len(pidsStartNextBlock) > 0 { + for _, l := range v.eventListeners { + l.OnProcessesStart(pidsStartNextBlock) + } + } + // Notify listeners about the commit state + for _, l := range v.eventListeners { + if err := l.Commit(height); err != nil { + log.Warnf("event callback error on commit: %v", err) + } + } + + // Commit the statedb tx + // Note that we need to commit the tx after calling listeners, because + // the listeners may need to get the previous (not committed) state. v.tx.Lock() err := func() error { var err error @@ -401,18 +418,6 @@ func (v *State) Save() ([]byte, error) { if err != nil { return nil, err } - // Notify listeners about processes that start in the next block. - if len(pidsStartNextBlock) > 0 { - for _, l := range v.eventListeners { - l.OnProcessesStart(pidsStartNextBlock) - } - } - // Notify listeners about the commit state - for _, l := range v.eventListeners { - if err := l.Commit(height); err != nil { - log.Warnf("event callback error on commit: %v", err) - } - } // Update the main state tree mainTreeView, err := v.store.TreeView(nil) diff --git a/vochain/state/votes.go b/vochain/state/votes.go index 335ef9e6b..26c012f49 100644 --- a/vochain/state/votes.go +++ b/vochain/state/votes.go @@ -235,6 +235,7 @@ func (s *State) Vote(processID, nullifier []byte, committed bool) (*models.State } else if err != nil { return nil, err } + sdbVoteBytes, err := votesTree.Get(vid) if errors.Is(err, arbo.ErrKeyNotFound) { return nil, ErrVoteNotFound From 753fe1f757242a02247bf597036c9360d37edba8 Mon Sep 17 00:00:00 2001 From: p4u Date: Fri, 1 Sep 2023 23:15:53 +0200 Subject: [PATCH 15/15] arbo: add a test and benchmark for comparing Add and AddBatch Ensure we produce the same root using both methods. Also update the dev genesis file. --- tree/arbo/addbatch_test.go | 2 +- tree/arbo/tree_test.go | 210 +++++++++++++++++++++++++++++++++++-- vochain/genesis/genesis.go | 4 +- vochain/state/sik_test.go | 2 +- 4 files changed, 208 insertions(+), 10 deletions(-) diff --git a/tree/arbo/addbatch_test.go b/tree/arbo/addbatch_test.go index e662191d5..40290c562 100644 --- a/tree/arbo/addbatch_test.go +++ b/tree/arbo/addbatch_test.go @@ -738,7 +738,7 @@ func TestFlp2(t *testing.T) { } func TestAddBatchBench(t *testing.T) { - nLeafs := 50_000 + nLeafs := 5_000 printTestContext("TestAddBatchBench: ", nLeafs, "Blake2b", "pebbledb") // prepare inputs diff --git a/tree/arbo/tree_test.go b/tree/arbo/tree_test.go index 12ef9fa13..1e32dca30 100644 --- a/tree/arbo/tree_test.go +++ b/tree/arbo/tree_test.go @@ -947,7 +947,7 @@ func benchmarkAdd(b *testing.B, hashFunc HashFunction, ks, vs [][]byte) { func TestDiskSizeBench(t *testing.T) { c := qt.New(t) - nLeafs := 500 + nLeafs := 1000 printTestContext("TestDiskSizeBench: ", nLeafs, "Blake2b", "pebble") // prepare inputs @@ -994,8 +994,9 @@ func TestDiskSizeBench(t *testing.T) { tree.dbg.print(" ") size, err := dirSize(dbDir) c.Assert(err, qt.IsNil) + dbItemsAdd := countDBitems() printRes(" Disk size (add)", fmt.Sprintf("%d MiB", size/(1024*1024))) - printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + printRes(" DB items", fmt.Sprintf("%d", dbItemsAdd)) // delete the leafs start = time.Now() @@ -1010,8 +1011,13 @@ func TestDiskSizeBench(t *testing.T) { tree.dbg.print(" ") size, err = dirSize(dbDir) c.Assert(err, qt.IsNil) + dbItemsDelete := countDBitems() printRes(" Disk size (delete)", fmt.Sprintf("%d MiB", size/(1024*1024))) - printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + printRes(" DB items", fmt.Sprintf("%d", dbItemsDelete)) + + if dbItemsDelete+(dbItemsDelete/4) >= dbItemsAdd { + t.Fatal("DB items after delete is too big") + } // add the leafs deleted again start = time.Now() @@ -1019,15 +1025,18 @@ func TestDiskSizeBench(t *testing.T) { for i := 0; i < len(ks)/2; i++ { err = tree.AddWithTx(tx, ks[i], vs[i]) c.Assert(err, qt.IsNil) - //fmt.Printf("added %x\n", ks[i]) } c.Assert(tx.Commit(), qt.IsNil) printRes(" Add2 loop", time.Since(start)) tree.dbg.print(" ") size, err = dirSize(dbDir) + dbItemsAdd2 := countDBitems() c.Assert(err, qt.IsNil) printRes(" Disk size (add2)", fmt.Sprintf("%d MiB", size/(1024*1024))) - printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + printRes(" DB items", fmt.Sprintf("%d", dbItemsAdd2)) + if dbItemsAdd2 != dbItemsAdd { + t.Fatal("DB items after add2 is not equal to DB items after add") + } // update the leafs start = time.Now() @@ -1041,9 +1050,13 @@ func TestDiskSizeBench(t *testing.T) { printRes(" Update loop", time.Since(start)) tree.dbg.print(" ") size, err = dirSize(dbDir) + dbItemsUpdate := countDBitems() c.Assert(err, qt.IsNil) printRes(" Disk size (update)", fmt.Sprintf("%d MiB", size/(1024*1024))) - printRes(" DB items", fmt.Sprintf("%d", countDBitems())) + printRes(" DB items", fmt.Sprintf("%d", dbItemsUpdate)) + if dbItemsUpdate != dbItemsAdd { + t.Fatal("DB items after update is not equal to DB items after add") + } start = time.Now() c.Assert(tree.db.Compact(), qt.IsNil) @@ -1239,3 +1252,188 @@ func TestTreeWithSingleLeaf(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(hex.EncodeToString(root), qt.DeepEquals, "0000000000000000000000000000000000000000000000000000000000000000") } + +// BenchmarkAddVsAddBatch benchmarks the Add and AddBatch functions and compare they produce the same output. +// go test -run=NONE -bench=BenchmarkAddVsAddBatch -benchmem +func BenchmarkAddVsAddBatch(b *testing.B) { + nLeaves := 200000 + bLen := 32 + // Prepare the initial set of keys and values + ks1, vs1 := generateKV(bLen, nLeaves/2, 0) + // Prepare the second set of keys and values + ks2, vs2 := generateKV(bLen, nLeaves/2, nLeaves) + + var rootAdd, rootAddBatch []byte + + b.Run("Add", func(b *testing.B) { + database := metadb.NewTest(b) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionBlake2b}) + if err != nil { + b.Fatal(err) + } + + // Add the first set of key/values + for i := 0; i < len(ks1); i++ { + if err := tree.Add(ks1[i], vs1[i]); err != nil { + b.Fatal(err) + } + } + + // Execute some deletes + for i := 0; i < len(ks1)/4; i++ { + if err := tree.Delete(ks1[i]); err != nil { + b.Fatal(err) + } + } + + // Execute some updates + for i := len(ks1)/4 + 1; i < len(ks1)/2; i++ { + if err := tree.Update(ks1[i], vs2[i]); err != nil { + b.Fatal(err) + } + } + + // Add the second set of key/values + for i := 0; i < len(ks2); i++ { + if err := tree.Add(ks2[i], vs2[i]); err != nil { + b.Fatal(err) + } + } + + rootAdd, err = tree.Root() + if err != nil { + b.Fatal(err) + } + }) + + b.Run("AddBatch", func(b *testing.B) { + database := metadb.NewTest(b) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionBlake2b}) + if err != nil { + b.Fatal(err) + } + + // Add the first set of key/values + if _, err := tree.AddBatch(ks1, vs1); err != nil { + b.Fatal(err) + } + + // Execute some deletes + for i := 0; i < len(ks1)/4; i++ { + if err := tree.Delete(ks1[i]); err != nil { + b.Fatal(err) + } + } + + // Execute some updates + for i := len(ks1)/4 + 1; i < len(ks1)/2; i++ { + if err := tree.Update(ks1[i], vs2[i]); err != nil { + b.Fatal(err) + } + } + // Add the second set of key/values + if _, err := tree.AddBatch(ks2, vs2); err != nil { + b.Fatal(err) + } + + rootAddBatch, err = tree.Root() + if err != nil { + b.Fatal(err) + } + }) + + // Verify that the roots are the same + if !bytes.Equal(rootAdd, rootAddBatch) { + b.Fatalf("Roots are different: Add root = %x, AddBatch root = %x", rootAdd, rootAddBatch) + } +} + +// generateKV generates a slice of keys and a slice of values. +// The keys are byte representations of sequential integers, and the values are random bytes. +func generateKV(bLen, count, offset int) ([][]byte, [][]byte) { + var keys, values [][]byte + + for i := 0; i < count; i++ { + // Convert integer i to a byte slice + key := BigIntToBytes(bLen, big.NewInt(int64(offset+i))) + value := randomBytes(bLen) + + keys = append(keys, key) + values = append(values, value) + } + + return keys, values +} + +// TestAddVsAddBatch tests that the Add and AddBatch functions produce the same output. +func TestAddVsAddBatch(t *testing.T) { + c := qt.New(t) + + // Create a new tree + database := metadb.NewTest(t) + tree, err := NewTree(Config{Database: database, MaxLevels: 256, HashFunction: HashFunctionBlake2b}) + c.Assert(err, qt.IsNil) + + // 1. Generate initial key-values and add them + keys1, values1 := generateKV(32, 1000, 0) + keys2, values2 := generateKV(32, 1000, 1000) + + for i, key := range keys1 { + err := tree.Add(key, values1[i]) + c.Assert(err, qt.IsNil) + } + + // 2. Execute some Deletes and Updates + for i := 0; i < len(keys1)/2; i++ { + err := tree.Delete(keys1[i]) + c.Assert(err, qt.IsNil) + } + + for i := len(keys1) / 2; i < len(keys1); i++ { + err := tree.Update(keys1[i], values2[i]) + c.Assert(err, qt.IsNil) + } + + // 3. Add the second set of key-values + for i, key := range keys2 { + err := tree.Add(key, values2[i]) + c.Assert(err, qt.IsNil) + } + + // Store the root hash after using Add method + addRoot, err := tree.Root() + c.Assert(err, qt.IsNil) + + // Repeat the operations using AddBatch + database2 := metadb.NewTest(t) + tree2, err := NewTree(Config{Database: database2, MaxLevels: 256, HashFunction: HashFunctionBlake2b}) + c.Assert(err, qt.IsNil) + + // 1. Add initial key-values + inv, err := tree2.AddBatch(keys1, values1) + c.Assert(err, qt.IsNil) + c.Assert(inv, qt.HasLen, 0) + + // 2. Execute some Deletes and Updates + for i := 0; i < len(keys1)/2; i++ { + err := tree2.Delete(keys1[i]) + c.Assert(err, qt.IsNil) + } + + for i := len(keys1) / 2; i < len(keys1); i++ { + err := tree2.Update(keys1[i], values2[i]) + c.Assert(err, qt.IsNil) + } + + // 3. Add more key-values + inv, err = tree2.AddBatch(keys2, values2) + c.Assert(err, qt.IsNil) + c.Assert(inv, qt.HasLen, 0) + + // Get root after AddBatch method + addBatchRoot, err := tree2.Root() + c.Assert(err, qt.IsNil) + + // Compare the root hashes of the two trees + c.Assert(addRoot, qt.DeepEquals, addBatchRoot) +} diff --git a/vochain/genesis/genesis.go b/vochain/genesis/genesis.go index 8139e77d5..3d05b0fd2 100644 --- a/vochain/genesis/genesis.go +++ b/vochain/genesis/genesis.go @@ -40,8 +40,8 @@ var Genesis = map[string]VochainGenesis{ } var devGenesis = GenesisDoc{ - GenesisTime: time.Date(2023, time.August, 1, 1, 0, 0, 0, time.UTC), - ChainID: "vocdoni-dev-16", + GenesisTime: time.Date(2023, time.September, 4, 1, 0, 0, 0, time.UTC), + ChainID: "vocdoni-dev-18", ConsensusParams: &ConsensusParams{ Block: BlockParams{ MaxBytes: 2097152, diff --git a/vochain/state/sik_test.go b/vochain/state/sik_test.go index a2531c6e7..6cff2c78b 100644 --- a/vochain/state/sik_test.go +++ b/vochain/state/sik_test.go @@ -165,7 +165,7 @@ func TestAssignSIKToElectionAndPurge(t *testing.T) { c.Assert(err, qt.IsNil) _, err = s.NoState(true).Get(key) c.Assert(err, qt.IsNotNil) - sik, err = s.SIKFromAddress(testAccount.Address()) + _, err = s.SIKFromAddress(testAccount.Address()) c.Assert(err, qt.IsNotNil, qt.Commentf("SIK should be deleted")) }