From f79d57b25864d261f4f4c2d3317af826a611388c Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Tue, 2 Jul 2024 17:02:11 +0200 Subject: [PATCH 1/7] api: re-enable autoswag since upstream bug is fixed --- api/api.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/api.go b/api/api.go index 7adc16352..835a2c868 100644 --- a/api/api.go +++ b/api/api.go @@ -1,10 +1,7 @@ package api -////// Disabled autoswag due to https://github.com/swaggo/swag/issues/1267 -////// TODO: re-enable when a fixed swaggo/swag is released -////// and remove the workaround done by @selankon on docs/models/models.go -////go:generate go run go.vocdoni.io/dvote/api/autoswag -//go:generate go run github.com/swaggo/swag/cmd/swag@v1.8.10 fmt +//go:generate go run go.vocdoni.io/dvote/api/autoswag +//go:generate go run github.com/swaggo/swag/cmd/swag@v1.16.3 fmt import ( "fmt" From cfc11733368022a998355551c20a633081a32d00 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH 2/7] indexer: CreateBlock during Commit instead of OnBeginBlock since the CreateBlock now fetches the whole block, we need app.NodeClient initialized in all nodes (including seeds), else seeds panic on first Commit --- vochain/app.go | 4 ---- vochain/appsetup.go | 3 --- vochain/indexer/block.go | 17 ----------------- vochain/indexer/indexer.go | 11 +++++++++++ vochain/keykeeper/keykeeper.go | 3 --- .../offchaindatahandler/offchaindatahandler.go | 1 - vochain/state/eventlistener.go | 15 --------------- vochain/state/state_test.go | 1 - 8 files changed, 11 insertions(+), 44 deletions(-) diff --git a/vochain/app.go b/vochain/app.go index 7e0107537..f6691be61 100644 --- a/vochain/app.go +++ b/vochain/app.go @@ -290,10 +290,6 @@ func (app *BaseApplication) beginBlock(t time.Time, height uint32) { app.State.SetHeight(height) go app.State.CachePurge(height) - app.State.OnBeginBlock(vstate.BeginBlock{ - Height: int64(height), - Time: t, - }) } // endBlock is called at the end of every block. diff --git a/vochain/appsetup.go b/vochain/appsetup.go index fb52f66ed..6726b8b21 100644 --- a/vochain/appsetup.go +++ b/vochain/appsetup.go @@ -25,9 +25,6 @@ func (app *BaseApplication) SetNode(vochaincfg *config.VochainCfg) error { if app.Node, err = newTendermint(app, vochaincfg); err != nil { return fmt.Errorf("could not set tendermint node service: %s", err) } - if vochaincfg.IsSeedNode { - return nil - } // Note that cometcli.New logs any error rather than returning it. app.NodeClient = cometcli.New(app.Node) return nil diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index e40a5fdfb..8c9d76a22 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,28 +6,11 @@ import ( "errors" "fmt" "time" - - "go.vocdoni.io/dvote/log" - indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" - "go.vocdoni.io/dvote/vochain/state" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. var ErrBlockNotFound = fmt.Errorf("block not found") -func (idx *Indexer) OnBeginBlock(bb state.BeginBlock) { - idx.blockMu.Lock() - defer idx.blockMu.Unlock() - queries := idx.blockTxQueries() - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - }); err != nil { - log.Errorw(err, "cannot index new block") - } -} - // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index c0b6d0289..a1b1fc9ff 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -414,6 +414,17 @@ func (idx *Indexer) Commit(height uint32) error { queries := idx.blockTxQueries() ctx := context.TODO() + // index the new block + bb := idx.App.GetBlockByHeight(int64(height)) + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + Height: bb.Height, + Time: bb.Time, + DataHash: nonNullBytes(bb.DataHash), + // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? + }); err != nil { + log.Errorw(err, "cannot index new block") + } + for _, pidStr := range updateProcs { pid := types.ProcessID(pidStr) if err := idx.updateProcess(ctx, queries, pid); err != nil { diff --git a/vochain/keykeeper/keykeeper.go b/vochain/keykeeper/keykeeper.go index 7f46f654d..4a4425cd1 100644 --- a/vochain/keykeeper/keykeeper.go +++ b/vochain/keykeeper/keykeeper.go @@ -268,9 +268,6 @@ func (*KeyKeeper) OnVote(_ *state.Vote, _ int32) {} // OnNewTx is not used by the KeyKeeper func (*KeyKeeper) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -// OnBeginBlock is not used by the KeyKeeper -func (*KeyKeeper) OnBeginBlock(_ state.BeginBlock) {} - // OnCensusUpdate is not used by the KeyKeeper func (*KeyKeeper) OnCensusUpdate(_, _ []byte, _ string, _ uint64) {} diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..4bc3be6b4 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -166,7 +166,6 @@ func (d *OffChainDataHandler) OnSetAccount(_ []byte, account *state.Account) { func (*OffChainDataHandler) OnCancel(_ []byte, _ int32) {} func (*OffChainDataHandler) OnVote(_ *state.Vote, _ int32) {} func (*OffChainDataHandler) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*OffChainDataHandler) OnBeginBlock(state.BeginBlock) {} func (*OffChainDataHandler) OnProcessKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnRevealKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} diff --git a/vochain/state/eventlistener.go b/vochain/state/eventlistener.go index 1c2d05281..7b599702b 100644 --- a/vochain/state/eventlistener.go +++ b/vochain/state/eventlistener.go @@ -1,8 +1,6 @@ package state import ( - "time" - "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" ) @@ -32,7 +30,6 @@ type EventListener interface { OnSpendTokens(addr []byte, txType models.TxType, cost uint64, reference string) OnCensusUpdate(pid, censusRoot []byte, censusURI string, censusSize uint64) Commit(height uint32) (err error) - OnBeginBlock(BeginBlock) Rollback() } @@ -46,15 +43,3 @@ func (v *State) AddEventListener(l EventListener) { func (v *State) CleanEventListeners() { v.eventListeners = nil } - -type BeginBlock struct { - Height int64 - Time time.Time - DataHash []byte -} - -func (v *State) OnBeginBlock(bb BeginBlock) { - for _, l := range v.eventListeners { - l.OnBeginBlock(bb) - } -} diff --git a/vochain/state/state_test.go b/vochain/state/state_test.go index e2b76c685..b36dc7722 100644 --- a/vochain/state/state_test.go +++ b/vochain/state/state_test.go @@ -182,7 +182,6 @@ type Listener struct { func (*Listener) OnVote(_ *Vote, _ int32) {} func (*Listener) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*Listener) OnBeginBlock(BeginBlock) {} func (*Listener) OnProcess(_ *models.Process, _ int32) {} func (*Listener) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} func (*Listener) OnProcessDurationChange(_ []byte, _ uint32, _ int32) {} From 28d359751378a2b4092e6679ea4d31525837f7e6 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:55:09 +0200 Subject: [PATCH 3/7] indexer: add block details chain_id, proposer_address and last_block_hash * add method BlockByHeight --- vochain/indexer/block.go | 17 ++++++++-- vochain/indexer/db/blocks.sql.go | 33 ++++++++++++++----- vochain/indexer/db/models.go | 9 +++-- vochain/indexer/indexer.go | 19 ++++++----- vochain/indexer/indexertypes/block.go | 32 ++++++++++++++++++ .../0013_alter_columns_table_blocks.sql | 13 ++++++++ vochain/indexer/queries/blocks.sql | 4 +-- 7 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 vochain/indexer/indexertypes/block.go create mode 100644 vochain/indexer/migrations/0013_alter_columns_table_blocks.sql diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index 8c9d76a22..8fc43c632 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "time" + + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. @@ -13,12 +15,21 @@ var ErrBlockNotFound = fmt.Errorf("block not found") // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { + block, err := idx.BlockByHeight(height) + if err != nil { + return time.Time{}, err + } + return block.Time, nil +} + +// BlockByHeight returns the available information of the block at the given height +func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) if err != nil { if errors.Is(err, sql.ErrNoRows) { - return time.Time{}, ErrBlockNotFound + return nil, ErrBlockNotFound } - return time.Time{}, err + return nil, err } - return block.Time, nil + return indexertypes.BlockFromDB(&block), nil } diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ffa3fa896..ce2f4cd03 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -13,24 +13,34 @@ import ( const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ) ` type CreateBlockParams struct { - Height int64 - Time time.Time - DataHash []byte + ChainID string + Height int64 + Time time.Time + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.Result, error) { - return q.exec(ctx, q.createBlockStmt, createBlock, arg.Height, arg.Time, arg.DataHash) + return q.exec(ctx, q.createBlockStmt, createBlock, + arg.ChainID, + arg.Height, + arg.Time, + arg.Hash, + arg.ProposerAddress, + arg.LastBlockHash, + ) } const getBlock = `-- name: GetBlock :one -SELECT height, time, data_hash FROM blocks +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` @@ -38,6 +48,13 @@ LIMIT 1 func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) var i Block - err := row.Scan(&i.Height, &i.Time, &i.DataHash) + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) return i, err } diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index a1566a32a..815ddc5ab 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -17,9 +17,12 @@ type Account struct { } type Block struct { - Height int64 - Time time.Time - DataHash []byte + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } type Process struct { diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index a1b1fc9ff..e9ac4ddb0 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -415,14 +415,17 @@ func (idx *Indexer) Commit(height uint32) error { ctx := context.TODO() // index the new block - bb := idx.App.GetBlockByHeight(int64(height)) - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? - }); err != nil { - log.Errorw(err, "cannot index new block") + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } } for _, pidStr := range updateProcs { diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go new file mode 100644 index 000000000..9864f4101 --- /dev/null +++ b/vochain/indexer/indexertypes/block.go @@ -0,0 +1,32 @@ +package indexertypes + +import ( + "time" + + "go.vocdoni.io/dvote/types" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" +) + +// Block represents a block handled by the Vochain. +// The indexer Block data type is different from the vochain state data type +// since it is optimized for querying purposes and not for keeping a shared consensus state. +type Block struct { + ChainID string `json:"chainId"` + Height int64 `json:"height"` + Time time.Time `json:"time"` + Hash types.HexBytes `json:"hash"` + ProposerAddress types.HexBytes `json:"proposer"` + LastBlockHash types.HexBytes `json:"lastBlockHash"` +} + +// BlockFromDB converts the indexerdb.Block into a Block +func BlockFromDB(dbblock *indexerdb.Block) *Block { + return &Block{ + ChainID: dbblock.ChainID, + Height: dbblock.Height, + Time: dbblock.Time, + Hash: nonEmptyBytes(dbblock.Hash), + ProposerAddress: nonEmptyBytes(dbblock.ProposerAddress), + LastBlockHash: nonEmptyBytes(dbblock.LastBlockHash), + } +} diff --git a/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql b/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql new file mode 100644 index 000000000..c83c32c06 --- /dev/null +++ b/vochain/indexer/migrations/0013_alter_columns_table_blocks.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE blocks DROP COLUMN data_hash; +ALTER TABLE blocks ADD COLUMN chain_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE blocks ADD COLUMN hash BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN proposer_address BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN last_block_hash BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE blocks ADD COLUMN data_hash BLOB NOT NULL; +ALTER TABLE blocks DROP COLUMN chain_id; +ALTER TABLE blocks DROP COLUMN hash; +ALTER TABLE blocks DROP COLUMN proposer_address; +ALTER TABLE blocks DROP COLUMN last_block_hash; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 577e875b5..90cf2a1a7 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -1,8 +1,8 @@ -- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ); -- name: GetBlock :one From d124a92179164ac2a895649b120bbaa1b15e7be8 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 8 Jul 2024 13:56:03 +0200 Subject: [PATCH 4/7] indexer: add GetBlockByHash, and rename GetBlock -> GetBlockByHeight --- vochain/indexer/block.go | 14 +++- vochain/indexer/db/blocks.sql.go | 26 +++++++- vochain/indexer/db/db.go | 104 ++++++++++++++++------------- vochain/indexer/queries/blocks.sql | 7 +- 4 files changed, 99 insertions(+), 52 deletions(-) diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index 8fc43c632..1e817aa53 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -24,7 +24,19 @@ func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { // BlockByHeight returns the available information of the block at the given height func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { - block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) + block, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), height) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockByHeight returns the available information of the block with the given hash +func (idx *Indexer) BlockByHash(hash []byte) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHash(context.TODO(), hash) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrBlockNotFound diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ce2f4cd03..3c7f022fe 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -39,14 +39,34 @@ func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.R ) } -const getBlock = `-- name: GetBlock :one +const getBlockByHash = `-- name: GetBlockByHash :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks +WHERE hash = ? +LIMIT 1 +` + +func (q *Queries) GetBlockByHash(ctx context.Context, hash []byte) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHashStmt, getBlockByHash, hash) + var i Block + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) + return i, err +} + +const getBlockByHeight = `-- name: GetBlockByHeight :one SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` -func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { - row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) +func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHeightStmt, getBlockByHeight, height) var i Block err := row.Scan( &i.Height, diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 9db9e8cdf..04b538964 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -60,8 +60,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.createVoteStmt, err = db.PrepareContext(ctx, createVote); err != nil { return nil, fmt.Errorf("error preparing query CreateVote: %w", err) } - if q.getBlockStmt, err = db.PrepareContext(ctx, getBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetBlock: %w", err) + if q.getBlockByHashStmt, err = db.PrepareContext(ctx, getBlockByHash); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHash: %w", err) + } + if q.getBlockByHeightStmt, err = db.PrepareContext(ctx, getBlockByHeight); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHeight: %w", err) } if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) @@ -209,9 +212,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing createVoteStmt: %w", cerr) } } - if q.getBlockStmt != nil { - if cerr := q.getBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getBlockStmt: %w", cerr) + if q.getBlockByHashStmt != nil { + if cerr := q.getBlockByHashStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHashStmt: %w", cerr) + } + } + if q.getBlockByHeightStmt != nil { + if cerr := q.getBlockByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHeightStmt: %w", cerr) } } if q.getEntityCountStmt != nil { @@ -400,7 +408,8 @@ type Queries struct { createTokenTransferStmt *sql.Stmt createTransactionStmt *sql.Stmt createVoteStmt *sql.Stmt - getBlockStmt *sql.Stmt + getBlockByHashStmt *sql.Stmt + getBlockByHeightStmt *sql.Stmt getEntityCountStmt *sql.Stmt getLastTransactionsStmt *sql.Stmt getListAccountsStmt *sql.Stmt @@ -432,47 +441,48 @@ type Queries struct { func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getLastTransactionsStmt: q.getLastTransactionsStmt, - getListAccountsStmt: q.getListAccountsStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenFeesStmt: q.getTokenFeesStmt, - getTokenFeesByFromAccountStmt: q.getTokenFeesByFromAccountStmt, - getTokenFeesByReferenceStmt: q.getTokenFeesByReferenceStmt, - getTokenFeesByTxTypeStmt: q.getTokenFeesByTxTypeStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTokenTransfersByFromAccountStmt: q.getTokenTransfersByFromAccountStmt, - getTokenTransfersByToAccountStmt: q.getTokenTransfersByToAccountStmt, - getTransactionStmt: q.getTransactionStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockByHashStmt: q.getBlockByHashStmt, + getBlockByHeightStmt: q.getBlockByHeightStmt, + getEntityCountStmt: q.getEntityCountStmt, + getLastTransactionsStmt: q.getLastTransactionsStmt, + getListAccountsStmt: q.getListAccountsStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenFeesStmt: q.getTokenFeesStmt, + getTokenFeesByFromAccountStmt: q.getTokenFeesByFromAccountStmt, + getTokenFeesByReferenceStmt: q.getTokenFeesByReferenceStmt, + getTokenFeesByTxTypeStmt: q.getTokenFeesByTxTypeStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTokenTransfersByFromAccountStmt: q.getTokenTransfersByFromAccountStmt, + getTokenTransfersByToAccountStmt: q.getTokenTransfersByToAccountStmt, + getTransactionStmt: q.getTransactionStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, - getVoteStmt: q.getVoteStmt, - searchEntitiesStmt: q.searchEntitiesStmt, - searchProcessesStmt: q.searchProcessesStmt, - searchVotesStmt: q.searchVotesStmt, - setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, - setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, - updateProcessEndDateStmt: q.updateProcessEndDateStmt, - updateProcessFromStateStmt: q.updateProcessFromStateStmt, - updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, - updateProcessResultsStmt: q.updateProcessResultsStmt, + getVoteStmt: q.getVoteStmt, + searchEntitiesStmt: q.searchEntitiesStmt, + searchProcessesStmt: q.searchProcessesStmt, + searchVotesStmt: q.searchVotesStmt, + setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, + setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, + updateProcessEndDateStmt: q.updateProcessEndDateStmt, + updateProcessFromStateStmt: q.updateProcessFromStateStmt, + updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, + updateProcessResultsStmt: q.updateProcessResultsStmt, } } diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 90cf2a1a7..31bf9da79 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -5,7 +5,12 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ); --- name: GetBlock :one +-- name: GetBlockByHeight :one SELECT * FROM blocks WHERE height = ? LIMIT 1; + +-- name: GetBlockByHash :one +SELECT * FROM blocks +WHERE hash = ? +LIMIT 1; From 3b426405ba6c2f60602b539a712867ca06e04c0d Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 8 Jul 2024 14:13:19 +0200 Subject: [PATCH 5/7] api: fetch blocks from indexer rather than app BlockStore --- api/chain.go | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/api/chain.go b/api/chain.go index a67af1d7a..262debb71 100644 --- a/api/chain.go +++ b/api/chain.go @@ -13,7 +13,6 @@ import ( "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/httprouter/apirest" - "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/genesis" @@ -766,18 +765,26 @@ func (a *API) chainBlockHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) if err != nil { return err } - tmblock := a.vocapp.GetBlockByHeight(int64(height)) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHeight(int64(height)) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) } block := &Block{ Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, } data, err := json.Marshal(block) if err != nil { @@ -801,18 +808,26 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - tmblock := a.vocapp.GetBlockByHash(hash) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHash(hash) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) } block := &Block{ Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, } data, err := json.Marshal(block) if err != nil { From 476f679ab0e1f420b569f3cd746c7e34c9a85b4f Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 8 Jul 2024 11:15:43 +0200 Subject: [PATCH 6/7] indexer: add raw_tx in transactions table, to index the raw proto.Marshal(tx.Tx) --- vochain/indexer/bench_test.go | 7 ++++++- vochain/indexer/db/models.go | 1 + vochain/indexer/db/transactions.sql.go | 18 ++++++++++++------ vochain/indexer/indexer_test.go | 1 + .../0014_alter_columns_table_transactions.sql | 5 +++++ vochain/indexer/queries/transactions.sql | 4 ++-- vochain/indexer/transaction.go | 9 +++++++++ 7 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 vochain/indexer/migrations/0014_alter_columns_table_transactions.sql diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 631e83205..e47a77c63 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -85,6 +85,7 @@ func BenchmarkIndexer(b *testing.B) { tx := &vochaintx.Tx{ TxID: rnd.Random32(), TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, } idx.OnNewTx(tx, height, txBlockIndex) curTxs = append(curTxs, tx) @@ -138,7 +139,11 @@ func BenchmarkFetchTx(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < numTxs; j++ { - idx.OnNewTx(&vochaintx.Tx{TxID: util.Random32()}, uint32(i), int32(j)) + idx.OnNewTx(&vochaintx.Tx{ + TxID: util.Random32(), + TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, + }, uint32(i), int32(j)) } err := idx.Commit(uint32(i)) qt.Assert(b, err, qt.IsNil) diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 815ddc5ab..e82ece4d8 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -81,4 +81,5 @@ type Transaction struct { BlockHeight int64 BlockIndex int64 Type string + RawTx []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index c3d6e0b45..8c34eff85 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -27,9 +27,9 @@ func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, raw_tx ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ? ) ` @@ -38,6 +38,7 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + RawTx []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -46,11 +47,12 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.RawTx, ) } const getLastTransactions = `-- name: GetLastTransactions :many -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx FROM transactions ORDER BY id DESC LIMIT ? OFFSET ? @@ -76,6 +78,7 @@ func (q *Queries) GetLastTransactions(ctx context.Context, arg GetLastTransactio &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, ); err != nil { return nil, err } @@ -91,7 +94,7 @@ func (q *Queries) GetLastTransactions(ctx context.Context, arg GetLastTransactio } const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx FROM transactions WHERE id = ? LIMIT 1 ` @@ -105,12 +108,13 @@ func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, er &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, ) return i, err } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx FROM transactions WHERE hash = ? LIMIT 1 ` @@ -124,12 +128,13 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, ) return i, err } const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -148,6 +153,7 @@ func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.RawTx, ) return i, err } diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index ce0b8c9c7..904a56a77 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1350,6 +1350,7 @@ func TestTxIndexer(t *testing.T) { idx.OnNewTx(&vochaintx.Tx{ TxID: getTxID(i, j), TxModelType: "setAccount", + Tx: &models.Tx{Payload: &models.Tx_SetAccount{}}, }, uint32(i), int32(j)) } } diff --git a/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql b/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql new file mode 100644 index 000000000..b1302265c --- /dev/null +++ b/vochain/indexer/migrations/0014_alter_columns_table_transactions.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN raw_tx; diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 5b5ec4698..6ec340cf3 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,8 +1,8 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, raw_tx ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ? ); -- name: GetTransaction :one diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 5d8097886..9761bf365 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -11,6 +11,7 @@ import ( indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" + "google.golang.org/protobuf/proto" ) // ErrTransactionNotFound is returned if the transaction is not found. @@ -84,12 +85,20 @@ func (idx *Indexer) GetLastTransactions(limit, offset int32) ([]*indexertypes.Tr func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { idx.blockMu.Lock() defer idx.blockMu.Unlock() + + rawtx, err := proto.Marshal(tx.Tx) + if err != nil { + log.Errorw(err, "indexer cannot marshal new transaction") + return + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + RawTx: rawtx, }); err != nil { log.Errorw(err, "cannot index new transaction") } From 268095060d600e757fd86fa88dcf35d310eb6578 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Tue, 18 Jun 2024 11:36:11 +0200 Subject: [PATCH 7/7] api: refactor pagination * indexer: rename GetListAccounts -> AccountsList * indexer: AccountsList, ProcessList and EntityList now return a TotalCount * indexer: EntityList inverted order of args (from, max) to be consistent with others * test: add TestAPIAccountsList and TestAPIElectionsList * api: unify hardcoded structs into a new types: * AccountsList * ElectionsList * OrganizationsList * CountResult * api: add `pagination` field to endpoints: * GET /elections * GET /accounts * GET /chain/organizations * api: refactor filtered endpoints to unify pagination logic (and add `pagination` field): * GET /accounts/{organizationID}/elections/status/{status}/page/{page} * GET /accounts/{organizationID}/elections/page/{page} * GET /elections/page/{page} * POST /elections/filter/page/{page} * GET /chain/organizations/page/{page} * POST /chain/organizations/filter/page/{page} * GET /accounts/page/{page} also, marked all of these endpoints as deprecated on swagger docs * api: return ErrPageNotFound on paginated endpoints, when page is negative or higher than last_page * api: deduplicate several code snippets, with marshalAndSend and parse* helpers * rename api.MaxPageSize -> api.ItemsPerPage * fixed lots of swagger docs --- api/accounts.go | 265 +++++++-------- api/api.go | 13 +- api/api_types.go | 47 ++- api/censuses.go | 10 +- api/chain.go | 271 +++++++-------- ...dler.md => electionListByFilterHandler.md} | 0 api/docs/models/models.go | 34 -- api/elections.go | 311 +++++++++++------- api/errors.go | 8 +- api/helpers.go | 121 +++++++ httprouter/message.go | 6 + test/api_test.go | 129 +++++++- test/apierror_test.go | 14 +- test/testcommon/testutil/apiclient.go | 15 +- vochain/indexer/db/account.sql.go | 27 +- vochain/indexer/db/models.go | 6 - vochain/indexer/db/processes.sql.go | 76 +++-- vochain/indexer/indexer.go | 23 +- vochain/indexer/indexer_test.go | 60 ++-- vochain/indexer/indexertypes/types.go | 5 + vochain/indexer/process.go | 52 +-- vochain/indexer/queries/account.sql | 9 +- vochain/indexer/queries/processes.sql | 38 ++- 23 files changed, 948 insertions(+), 592 deletions(-) rename api/docs/descriptions/{electionFilterPaginatedHandler.md => electionListByFilterHandler.md} (100%) diff --git a/api/accounts.go b/api/accounts.go index e832da44a..046fcd29f 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "strconv" "strings" "time" @@ -56,7 +55,7 @@ func (a *API) enableAccountHandlers() error { "/accounts/{organizationID}/elections/count", "GET", apirest.MethodAccessTypePublic, - a.electionCountHandler, + a.accountElectionsCountHandler, ); err != nil { return err } @@ -64,7 +63,7 @@ func (a *API) enableAccountHandlers() error { "/accounts/{organizationID}/elections/status/{status}/page/{page}", "GET", apirest.MethodAccessTypePublic, - a.electionListHandler, + a.accountElectionsListByStatusAndPageHandler, ); err != nil { return err } @@ -72,7 +71,7 @@ func (a *API) enableAccountHandlers() error { "/accounts/{organizationID}/elections/page/{page}", "GET", apirest.MethodAccessTypePublic, - a.electionListHandler, + a.accountElectionsListByPageHandler, ); err != nil { return err } @@ -112,6 +111,14 @@ func (a *API) enableAccountHandlers() error { "/accounts/page/{page}", "GET", apirest.MethodAccessTypePublic, + a.accountListByPageHandler, + ); err != nil { + return err + } + if err := a.Endpoint.RegisterMethod( + "/accounts", + "GET", + apirest.MethodAccessTypePublic, a.accountListHandler, ); err != nil { return err @@ -130,9 +137,9 @@ func (a *API) enableAccountHandlers() error { // @Produce json // @Param address path string true "Account address" // @Success 200 {object} Account +// @Success 200 {object} AccountMetadata // @Router /accounts/{address} [get] // @Router /accounts/{address}/metadata [get] -// @Success 200 {object} AccountMetadata func (a *API) accountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { if len(util.TrimHex(ctx.URLParam("address"))) != common.AddressLength*2 { return ErrAddressMalformed @@ -311,108 +318,78 @@ func (a *API) accountSetHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContex // @Tags Accounts // @Accept json // @Produce json -// @Success 200 {object} object{count=int} +// @Success 200 {object} CountResult // @Router /accounts/count [get] func (a *API) accountCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { count, err := a.indexer.CountTotalAccounts() if err != nil { return err } + return marshalAndSend(ctx, &CountResult{Count: count}) +} - data, err := json.Marshal( - struct { - Count uint64 `json:"count"` - }{Count: count}, +// accountElectionsListByPageHandler +// +// @Summary List organization elections (deprecated, uses url params) +// @Description List the elections of an organization (deprecated, in favor of /elections?page=xxx&organizationID=xxx) +// @Tags Accounts +// @Accept json +// @Produce json +// @Param organizationID path string true "Specific organizationID" +// @Param page path number true "Page" +// @Success 200 {object} ElectionsList +// @Router /accounts/{organizationID}/elections/page/{page} [get] +func (a *API) accountElectionsListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseElectionFilterParams( + ctx.URLParam(ParamPage), + "", + ctx.URLParam(ParamOrganizationID), + "", + "", ) if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) + return err } - return ctx.Send(data, apirest.HTTPstatusOK) + if params.OrganizationID == nil { + return ErrMissingParameter + } + + return a.sendElectionList(ctx, params) } -// electionListHandler +// accountElectionsListByStatusAndPageHandler // -// @Summary List organization elections -// @Description List the elections of an organization +// @Summary List organization elections by status (deprecated, uses url params) +// @Description List the elections of an organization by status (deprecated, in favor of /elections?page=xxx&organizationID=xxx&status=xxx) // @Tags Accounts // @Accept json // @Produce json // @Param organizationID path string true "Specific organizationID" -// @Param page path number true "Define de page number" -// @Success 200 {object} object{elections=[]ElectionSummary} -// @Router /accounts/{organizationID}/elections/page/{page} [get] -// /accounts/{organizationID}/elections/status/{status}/page/{page} [post] Endpoint docs generated on docs/models/model.go -func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - organizationID, err := hex.DecodeString(util.TrimHex(ctx.URLParam("organizationID"))) - if err != nil || organizationID == nil { - return ErrCantParseOrgID.Withf("%q", ctx.URLParam("organizationID")) - } - - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } +// @Param status path string true "Election status" Enums(ready, paused, canceled, ended, results) +// @Param page path number true "Page" +// @Success 200 {object} ElectionsList +// @Router /accounts/{organizationID}/elections/status/{status}/page/{page} [get] +func (a *API) accountElectionsListByStatusAndPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseElectionFilterParams( + ctx.URLParam(ParamPage), + ctx.URLParam(ParamStatus), + ctx.URLParam(ParamOrganizationID), + "", + "", + ) + if err != nil { + return err } - page = page * MaxPageSize - var pids [][]byte - switch ctx.URLParam("status") { - case "ready": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "READY", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - case "paused": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "PAUSED", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - case "canceled": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "CANCELED", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - case "ended", "results": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "RESULTS", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - pids2, err := a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "ENDED", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - pids = append(pids, pids2...) - case "": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "", false) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - default: - return ErrParamStatusMissing + if params.OrganizationID == nil || params.Status == "" { + return ErrMissingParameter } - elections := []*ElectionSummary{} - for _, pid := range pids { - procInfo, err := a.indexer.ProcessInfo(pid) - if err != nil { - return ErrCantFetchElection.WithErr(err) - } - summary := a.electionSummary(procInfo) - elections = append(elections, &summary) - } - data, err := json.Marshal(&Organization{ - Elections: elections, - }) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) + return a.sendElectionList(ctx, params) } -// electionCountHandler +// accountElectionsCountHandler // // @Summary Count organization elections // @Description Returns the number of elections for an organization @@ -420,13 +397,18 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex // @Accept json // @Produce json // @Param organizationID path string true "Specific organizationID" -// @Success 200 {object} object{count=number} +// @Success 200 {object} CountResult // @Router /accounts/{organizationID}/elections/count [get] -func (a *API) electionCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - organizationID, err := hex.DecodeString(util.TrimHex(ctx.URLParam("organizationID"))) - if err != nil || organizationID == nil { - return ErrCantParseOrgID.Withf("%q", ctx.URLParam("organizationID")) +func (a *API) accountElectionsCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + if ctx.URLParam(ParamOrganizationID) == "" { + return ErrMissingParameter + } + + organizationID, err := parseHexString(ctx.URLParam(ParamOrganizationID)) + if err != nil { + return err } + acc, err := a.vocapp.State.GetAccount(common.BytesToAddress(organizationID), true) if acc == nil { return ErrOrgNotFound @@ -434,15 +416,7 @@ func (a *API) electionCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPConte if err != nil { return err } - data, err := json.Marshal( - struct { - Count uint32 `json:"count"` - }{Count: acc.GetProcessIndex()}, - ) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) + return marshalAndSend(ctx, &CountResult{Count: uint64(acc.GetProcessIndex())}) } // tokenTransfersListHandler @@ -453,7 +427,7 @@ func (a *API) electionCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPConte // @Accept json // @Produce json // @Param accountID path string true "Specific accountID" -// @Param page path string true "Paginator page" +// @Param page path number true "Page" // @Success 200 {object} object{transfers=indexertypes.TokenTransfersAccount} // @Router /accounts/{accountID}/transfers/page/{page} [get] func (a *API) tokenTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { @@ -468,15 +442,13 @@ func (a *API) tokenTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTP if err != nil { return err } - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize - transfers, err := a.indexer.GetTokenTransfersByAccount(accountID, int32(page), MaxPageSize) + + transfers, err := a.indexer.GetTokenTransfersByAccount(accountID, int32(page*ItemsPerPage), ItemsPerPage) if err != nil { return ErrCantFetchTokenTransfers.WithErr(err) } @@ -499,7 +471,7 @@ func (a *API) tokenTransfersListHandler(_ *apirest.APIdata, ctx *httprouter.HTTP // @Accept json // @Produce json // @Param accountID path string true "Specific accountID" -// @Param page path string true "Paginator page" +// @Param page path number true "Page" // @Success 200 {object} object{fees=[]indexertypes.TokenFeeMeta} // @Router /accounts/{accountID}/fees/page/{page} [get] func (a *API) tokenFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { @@ -514,16 +486,12 @@ func (a *API) tokenFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) if err != nil { return err } - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize - fees, err := a.indexer.GetTokenFeesByFromAccount(accountID, int32(page), MaxPageSize) + fees, err := a.indexer.GetTokenFeesByFromAccount(accountID, int32(page*ItemsPerPage), ItemsPerPage) if err != nil { return ErrCantFetchTokenTransfers.WithErr(err) } @@ -545,8 +513,8 @@ func (a *API) tokenFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) // @Tags Accounts // @Accept json // @Produce json -// @Param accountID path string true "Specific accountID" -// @Success 200 {object} object{count=int} "Number of transaction sent and received for the account" +// @Param accountID path string true "Specific accountID" +// @Success 200 {object} CountResult "Number of transaction sent and received for the account" // @Router /accounts/{accountID}/transfers/count [get] func (a *API) tokenTransfersCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { accountID, err := hex.DecodeString(util.TrimHex(ctx.URLParam("accountID"))) @@ -565,16 +533,21 @@ func (a *API) tokenTransfersCountHandler(_ *apirest.APIdata, ctx *httprouter.HTT if err != nil { return err } - data, err := json.Marshal( - struct { - Count uint64 `json:"count"` - }{Count: count}, - ) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } + return marshalAndSend(ctx, &CountResult{Count: count}) +} - return ctx.Send(data, apirest.HTTPstatusOK) +// accountListByPageHandler +// +// @Summary List of the existing accounts (using url params) (deprecated) +// @Description Returns information (address, balance and nonce) of the existing accounts. (Deprecated, in favor of /accounts?page=) +// @Tags Accounts +// @Accept json +// @Produce json +// @Param page path number true "Page" +// @Success 200 {object} AccountsList +// @Router /accounts/page/{page} [get] +func (a *API) accountListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + return a.sendAccountList(ctx, ctx.URLParam(ParamPage)) } // accountListHandler @@ -584,30 +557,30 @@ func (a *API) tokenTransfersCountHandler(_ *apirest.APIdata, ctx *httprouter.HTT // @Tags Accounts // @Accept json // @Produce json -// @Param page path string true "Paginator page" -// @Success 200 {object} object{accounts=[]indexertypes.Account} -// @Router /accounts/page/{page} [get] +// @Param page query number false "Page" +// @Success 200 {object} AccountsList +// @Router /accounts [get] func (a *API) accountListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + return a.sendAccountList(ctx, ctx.QueryParam(ParamPage)) +} + +// sendAccountList sends a marshalled AccountsList over ctx.Send +func (a *API) sendAccountList(ctx *httprouter.HTTPContext, paramPage string) error { + page, err := parsePage(paramPage) + if err != nil { + return err } - page = page * MaxPageSize - accounts, err := a.indexer.GetListAccounts(int32(page), MaxPageSize) + accounts, total, err := a.indexer.AccountsList(page*ItemsPerPage, ItemsPerPage) if err != nil { - return ErrCantFetchTokenTransfers.WithErr(err) + return ErrIndexerQueryFailed.WithErr(err) } - data, err := json.Marshal( - struct { - Accounts []indexertypes.Account `json:"accounts"` - }{Accounts: accounts}, - ) + pagination, err := calculatePagination(page, total) if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) + return err } - return ctx.Send(data, apirest.HTTPstatusOK) + list := &AccountsList{ + Accounts: accounts, + Pagination: pagination, + } + return marshalAndSend(ctx, list) } diff --git a/api/api.go b/api/api.go index 835a2c868..48a9f1ca6 100644 --- a/api/api.go +++ b/api/api.go @@ -47,8 +47,17 @@ import ( // @securityDefinitions.basic BasicAuth -// MaxPageSize defines the maximum number of results returned by the paginated endpoints -const MaxPageSize = 10 +const ( + // ItemsPerPage defines how many items per page are returned by the paginated endpoints + ItemsPerPage = 10 + + // These consts define the keywords for both query (?param=) and url (/url/param/) params + ParamPage = "page" + ParamStatus = "status" + ParamOrganizationID = "organizationID" + ParamElectionID = "electionID" + ParamWithResults = "withResults" +) var ( ErrMissingModulesForHandler = fmt.Errorf("missing modules attached for enabling handler") diff --git a/api/api_types.go b/api/api_types.go index 65f20aa5f..3ab18cca2 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -12,18 +12,32 @@ import ( "google.golang.org/protobuf/encoding/protojson" ) -type Organization struct { - OrganizationID types.HexBytes `json:"organizationID,omitempty" ` - Elections []*ElectionSummary `json:"elections,omitempty"` - Organizations []*OrganizationList `json:"organizations,omitempty"` - Count *uint64 `json:"count,omitempty" example:"1"` +// CountResult wraps a count inside an object +type CountResult struct { + Count uint64 `json:"count" example:"10"` } -type OrganizationList struct { +// Pagination contains all the values needed for the UI to easily organize the returned data +type Pagination struct { + TotalItems uint64 `json:"total_items"` + PreviousPage *uint64 `json:"previous_page"` + CurrentPage uint64 `json:"current_page"` + NextPage *uint64 `json:"next_page"` + LastPage uint64 `json:"last_page"` +} + +type OrganizationSummary struct { OrganizationID types.HexBytes `json:"organizationID" example:"0x370372b92514d81a0e3efb8eba9d036ae0877653"` ElectionCount uint64 `json:"electionCount" example:"1"` } +// OrganizationList wraps the organizations list to consistently return the list inside an object, +// and return an empty object if the list does not contains any result +type OrganizationsList struct { + Organizations []OrganizationSummary `json:"organizations"` + Pagination *Pagination `json:"pagination"` +} + type ElectionSummary struct { ElectionID types.HexBytes `json:"electionId" ` OrganizationID types.HexBytes `json:"organizationId" ` @@ -37,6 +51,13 @@ type ElectionSummary struct { ChainID string `json:"chainId"` } +// ElectionsList wraps the elections list to consistently return the list inside an object, +// and return an empty object if the list does not contains any result +type ElectionsList struct { + Elections []ElectionSummary `json:"elections"` + Pagination *Pagination `json:"pagination"` +} + // ElectionResults is the struct used to wrap the results of an election type ElectionResults struct { // ABIEncoded is the abi encoded election results @@ -101,11 +122,16 @@ type ElectionDescription struct { } type ElectionFilter struct { - OrganizationID types.HexBytes `json:"organizationId,omitempty" ` - ElectionID types.HexBytes `json:"electionId,omitempty" ` + Page int `json:"page,omitempty"` + OrganizationID types.HexBytes `json:"organizationId,omitempty"` + ElectionID types.HexBytes `json:"electionId,omitempty"` WithResults *bool `json:"withResults,omitempty"` Status string `json:"status,omitempty"` } +type OrganizationFilter struct { + Page int `json:"page,omitempty"` + OrganizationID types.HexBytes `json:"organizationId,omitempty"` +} type Key struct { Index int `json:"index"` @@ -228,6 +254,11 @@ type Account struct { SIK types.HexBytes `json:"sik"` } +type AccountsList struct { + Accounts []indexertypes.Account `json:"accounts"` + Pagination *Pagination `json:"pagination"` +} + type AccountSet struct { TxPayload []byte `json:"txPayload,omitempty" swaggerignore:"true"` Metadata []byte `json:"metadata,omitempty" swaggerignore:"true"` diff --git a/api/censuses.go b/api/censuses.go index 7a61c2522..954e5e05b 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -631,8 +631,10 @@ func (a *API) censusDeleteHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCont // @Security BasicAuth // @Success 200 {object} object{census=object{censusID=string,uri=string}} "It return published censusID and the ipfs uri where its uploaded" // @Param censusID path string true "Census id" +// @Param root path string false "Specific root where to publish the census. Not required" // @Router /censuses/{censusID}/publish [post] // @Router /censuses/{censusID}/publish/async [post] +// @Router /censuses/{censusID}/publish/{root} [post] func (a *API) censusPublishHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { token, err := uuid.Parse(msg.AuthToken) if err != nil { @@ -957,7 +959,7 @@ func (a *API) censusVerifyHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCont // @Accept json // @Produce json // @Success 200 {object} object{valid=bool} -// @Router /censuses/list/ [get] +// @Router /censuses/list [get] func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { list, err := a.censusdb.List() if err != nil { @@ -979,7 +981,8 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) // @Produce json // @Param ipfs path string true "Export to IPFS. Blank to return the JSON file" // @Success 200 {object} object{valid=bool} -// @Router /censuses/export/{ipfs} [get] +// @Router /censuses/export/ipfs [get] +// @Router /censuses/export [get] func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs") buf := bytes.Buffer{} @@ -1012,7 +1015,8 @@ func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // @Accept json // @Produce json // @Success 200 {object} object{valid=bool} -// @Router /censuses/import/{ipfscid} [post] +// @Router /censuses/import/{ipfscid} [get] +// @Router /censuses/import [post] func (a *API) censusImportDBHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { ipfscid := ctx.URLParam("ipfscid") if ipfscid == "" { diff --git a/api/chain.go b/api/chain.go index a67af1d7a..276603f16 100644 --- a/api/chain.go +++ b/api/chain.go @@ -29,13 +29,21 @@ const ( func (a *API) enableChainHandlers() error { if err := a.Endpoint.RegisterMethod( - "/chain/organizations/page/{page}", + "/chain/organizations", "GET", apirest.MethodAccessTypePublic, a.organizationListHandler, ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/chain/organizations/page/{page}", + "GET", + apirest.MethodAccessTypePublic, + a.organizationListByPageHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/chain/organizations/count", "GET", @@ -168,7 +176,7 @@ func (a *API) enableChainHandlers() error { "/chain/organizations/filter/page/{page}", "POST", apirest.MethodAccessTypePublic, - a.chainOrganizationsFilterPaginatedHandler, + a.organizationListByFilterAndPageHandler, ); err != nil { return err } @@ -223,39 +231,99 @@ func (a *API) enableChainHandlers() error { // @Tags Chain // @Accept json // @Produce json -// @Param page path int true "Page number" -// @Success 200 {object} api.organizationListHandler.response -// @Router /chain/organizations/page/{page} [get] +// @Param page query number false "Page" +// @Param organizationID query string false "Filter by partial organizationID" +// @Success 200 {object} OrganizationsList +// @Router /chain/organizations [get] func (a *API) organizationListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } - } - page = page * MaxPageSize - organizations := []*OrganizationList{} + return a.sendOrganizationList(ctx, + ctx.QueryParam(ParamPage), + ctx.QueryParam(ParamOrganizationID), + ) +} - list := a.indexer.EntityList(MaxPageSize, page, "") - for _, org := range list { - organizations = append(organizations, &OrganizationList{ - OrganizationID: org.EntityID, - ElectionCount: uint64(org.ProcessCount), - }) +// organizationListByPageHandler +// +// @Summary List organizations (deprecated, uses url params) +// @Description List all organizations (deprecated, in favor of /chain/organizations?page=xxx) +// @Tags Chain +// @Accept json +// @Produce json +// @Param page path number true "Page" +// @Success 200 {object} OrganizationsList +// @Router /chain/organizations/page/{page} [get] +func (a *API) organizationListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + return a.sendOrganizationList(ctx, + ctx.URLParam(ParamPage), + "", + ) +} + +// organizationListByFilterAndPageHandler +// +// @Summary List organizations (filtered) (deprecated, uses url params) +// @Description Returns a list of organizations filtered by its partial id, paginated by the given page (deprecated, in favor of /chain/organizations?page=xxx&organizationID=xxx) +// @Tags Chain +// @Accept json +// @Produce json +// @Param organizationId body OrganizationFilter true "Partial organizationId to filter by" +// @Param page path number true "Page" +// @Success 200 {object} OrganizationsList +// @Router /chain/organizations/filter/page/{page} [post] +func (a *API) organizationListByFilterAndPageHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { + // get organizationId from the request body + body := &OrganizationFilter{} + if err := json.Unmarshal(msg.Data, &body); err != nil { + return ErrCantParseDataAsJSON.WithErr(err) + } + // check that at least one filter is set + if body.OrganizationID == nil { + return ErrMissingParameter } + return a.sendOrganizationList(ctx, + ctx.URLParam(ParamPage), + body.OrganizationID.String(), + ) +} - type response struct { - Organizations []*OrganizationList `json:"organizations"` +// sendOrganizationList produces a filtered, paginated OrganizationsList, +// and sends it marshalled over ctx.Send +// +// Errors returned are always of type APIerror. +func (a *API) sendOrganizationList(ctx *httprouter.HTTPContext, + paramPage, paramOrgID string, +) error { + page, err := parsePage(paramPage) + if err != nil { + return err } - data, err := json.Marshal(response{organizations}) + orgID, err := parseHexString(paramOrgID) if err != nil { return err } - return ctx.Send(data, apirest.HTTPstatusOK) + entities, total, err := a.indexer.EntityList(page*ItemsPerPage, ItemsPerPage, orgID.String()) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) + } + pagination, err := calculatePagination(page, total) + if err != nil { + return err + } + if total == 0 { + return ErrOrgNotFound + } + list := &OrganizationsList{ + Pagination: pagination, + } + for _, org := range entities { + list.Organizations = append(list.Organizations, OrganizationSummary{ + OrganizationID: org.EntityID, + ElectionCount: uint64(org.ProcessCount), + }) + } + return marshalAndSend(ctx, list) } // organizationCountHandler @@ -265,16 +333,11 @@ func (a *API) organizationListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo // @Tags Chain // @Accept json // @Produce json -// @Success 200 {object} object{count=int} "Number of registered organizations" +// @Success 200 {object} CountResult "Number of registered organizations" // @Router /chain/organizations/count [get] func (a *API) organizationCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { count := a.indexer.CountTotalEntities() - organization := &Organization{Count: &count} - data, err := json.Marshal(organization) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) + return marshalAndSend(ctx, &CountResult{Count: count}) } // chainInfoHandler @@ -503,20 +566,16 @@ func (a *API) chainTxCostHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext // @Tags Chain // @Accept json // @Produce json -// @Param page path int true "Page number" +// @Param page path number true "Page" // @Success 200 {object} api.chainTxListPaginated.response "It return a list of transactions references" // @Router /chain/transactions/page/{page} [get] func (a *API) chainTxListPaginated(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - page := 0 - if ctx.URLParam("page") != "" { - var err error - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return err - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - offset := int32(page * MaxPageSize) - refs, err := a.indexer.GetLastTransactions(MaxPageSize, offset) + + refs, err := a.indexer.GetLastTransactions(ItemsPerPage, int32(page*ItemsPerPage)) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -652,7 +711,7 @@ func (a *API) chainTxByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // @Accept json // @Produce json // @Param height path number true "Block height" -// @Param page path number true "Page to paginate" +// @Param page path number true "Page" // @Success 200 {object} []TransactionMetadata // @Router /chain/blocks/{height}/transactions/page/{page} [get] func (a *API) chainTxByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { @@ -670,17 +729,13 @@ func (a *API) chainTxByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon Transactions: make([]TransactionMetadata, 0), } - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber.WithErr(err) - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize count := 0 - for i := page; i < len(block.Txs); i++ { - if count >= MaxPageSize { + for i := page * ItemsPerPage; i < len(block.Txs); i++ { + if count >= ItemsPerPage { break } signedTx := new(models.SignedTx) @@ -821,59 +876,6 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo return ctx.Send(convertKeysToCamel(data), apirest.HTTPstatusOK) } -// chainOrganizationsFilterPaginatedHandler -// -// @Summary List organizations (filtered) -// @Description Returns a list of organizations filtered by its partial id, paginated by the given page -// @Tags Chain -// @Accept json -// @Produce json -// @Param organizationId body object{organizationId=string} true "Partial organizationId to filter by" -// @Param page path int true "Current page" -// @Success 200 {object} object{organizations=[]api.OrganizationList} -// @Router /chain/organizations/filter/page/{page} [post] -func (a *API) chainOrganizationsFilterPaginatedHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { - // get organizationId from the request body - requestData := struct { - OrganizationId string `json:"organizationId"` - }{} - if err := json.Unmarshal(msg.Data, &requestData); err != nil { - return ErrCantParseDataAsJSON.WithErr(err) - } - // get page - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber.WithErr(err) - } - } - page = page * MaxPageSize - - organizations := []*OrganizationList{} - // get matching organization ids from the indexer - matchingOrganizationIds := a.indexer.EntityList(MaxPageSize, page, util.TrimHex(requestData.OrganizationId)) - if len(matchingOrganizationIds) == 0 { - return ErrOrgNotFound - } - - for _, org := range matchingOrganizationIds { - organizations = append(organizations, &OrganizationList{ - OrganizationID: org.EntityID, - ElectionCount: uint64(org.ProcessCount), - }) - } - - data, err := json.Marshal(struct { - Organizations []*OrganizationList `json:"organizations"` - }{organizations}) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTransactionCountHandler // // @Summary Transactions count @@ -881,24 +883,14 @@ func (a *API) chainOrganizationsFilterPaginatedHandler(msg *apirest.APIdata, ctx // @Tags Chain // @Accept json // @Produce json -// @Success 200 {object} uint64 -// @Success 200 {object} object{count=number} +// @Success 200 {object} CountResult // @Router /chain/transactions/count [get] func (a *API) chainTxCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { count, err := a.indexer.CountTotalTransactions() if err != nil { return err } - data, err := json.Marshal( - struct { - Count uint64 `json:"count"` - }{Count: count}, - ) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - - return ctx.Send(data, apirest.HTTPstatusOK) + return marshalAndSend(ctx, &CountResult{Count: count}) } // chainListFeesHandler @@ -908,21 +900,16 @@ func (a *API) chainTxCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex // @Tags Accounts // @Accept json // @Produce json -// @Param page path string true "Paginator page" +// @Param page path number true "Page" // @Success 200 {object} object{fees=[]indexertypes.TokenFeeMeta} // @Router /chain/fees/page/{page} [get] func (a *API) chainListFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize - fees, err := a.indexer.GetTokenFees(int32(page), MaxPageSize) + fees, err := a.indexer.GetTokenFees(int32(page*ItemsPerPage), ItemsPerPage) if err != nil { return ErrCantFetchTokenTransfers.WithErr(err) } @@ -945,26 +932,21 @@ func (a *API) chainListFeesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPConte // @Accept json // @Produce json // @Param reference path string true "Reference filter" -// @Param page path string true "Paginator page" +// @Param page path number true "Page" // @Success 200 {object} object{fees=[]indexertypes.TokenFeeMeta} // @Router /chain/fees/reference/{reference}/page/{page} [get] func (a *API) chainListFeesByReferenceHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize reference := ctx.URLParam("reference") if reference == "" { return ErrMissingParameter } - fees, err := a.indexer.GetTokenFeesByReference(reference, int32(page), MaxPageSize) + fees, err := a.indexer.GetTokenFeesByReference(reference, int32(page*ItemsPerPage), ItemsPerPage) if err != nil { return ErrCantFetchTokenTransfers.WithErr(err) } @@ -987,26 +969,21 @@ func (a *API) chainListFeesByReferenceHandler(_ *apirest.APIdata, ctx *httproute // @Accept json // @Produce json // @Param type path string true "Type filter" -// @Param page path string true "Paginator page" +// @Param page path number true "Page" // @Success 200 {object} object{fees=[]indexertypes.TokenFeeMeta} // @Router /chain/fees/type/{type}/page/{page} [get] func (a *API) chainListFeesByTypeHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize typeFilter := ctx.URLParam("type") if typeFilter == "" { return ErrMissingParameter } - fees, err := a.indexer.GetTokenFeesByType(typeFilter, int32(page), MaxPageSize) + fees, err := a.indexer.GetTokenFeesByType(typeFilter, int32(page*ItemsPerPage), ItemsPerPage) if err != nil { return ErrCantFetchTokenTransfers.WithErr(err) } diff --git a/api/docs/descriptions/electionFilterPaginatedHandler.md b/api/docs/descriptions/electionListByFilterHandler.md similarity index 100% rename from api/docs/descriptions/electionFilterPaginatedHandler.md rename to api/docs/descriptions/electionListByFilterHandler.md diff --git a/api/docs/models/models.go b/api/docs/models/models.go index f9455a695..b0e8bf4fe 100644 --- a/api/docs/models/models.go +++ b/api/docs/models/models.go @@ -19,37 +19,3 @@ import ( // @Success 200 {object} models.Tx_SetKeykeeper func ChainTxHandler() { } - -// ElectionListByStatusHandler -// -// Add multiple router on swagger generation has this bug https://github.com/swaggo/swag/issues/1267 -// -// @Summary List organization elections by status -// @Description List the elections of an organization by status -// @Tags Accounts -// @Accept json -// @Produce json -// @Param organizationID path string true "Specific organizationID" -// @Param status path string true "Status of the election" Enums(ready, paused, canceled, ended, results) -// @Param page path number true "Define de page number" -// @Success 200 {object} object{elections=[]api.ElectionSummary} -// @Router /accounts/{organizationID}/elections/status/{status}/page/{page} [get] -func ElectionListByStatusHandler() { -} - -// CensusPublishRootHandler -// -// Add multiple router on swagger generation has this bug https://github.com/swaggo/swag/issues/1267 -// -// @Summary Publish census at root -// @Description.markdown censusPublishHandler -// @Tags Censuses -// @Accept json -// @Produce json -// @Security BasicAuth -// @Success 200 {object} object{census=object{censusID=string,uri=string}} "It return published censusID and the ipfs uri where its uploaded" -// @Param censusID path string true "Census id" -// @Param root path string true "Specific root where to publish the census. Not required" -// @Router /censuses/{censusID}/publish/{root} [post] -func CensusPublishRootHandler() { -} diff --git a/api/elections.go b/api/elections.go index 99ca26f63..df06c3c9f 100644 --- a/api/elections.go +++ b/api/elections.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "strconv" "strings" "time" @@ -34,7 +33,15 @@ func (a *API) enableElectionHandlers() error { "/elections/page/{page}", "GET", apirest.MethodAccessTypePublic, - a.electionFullListHandler, + a.electionListByPageHandler, + ); err != nil { + return err + } + if err := a.Endpoint.RegisterMethod( + "/elections", + "GET", + apirest.MethodAccessTypePublic, + a.electionListHandler, ); err != nil { return err } @@ -107,7 +114,15 @@ func (a *API) enableElectionHandlers() error { "/elections/filter/page/{page}", "POST", apirest.MethodAccessTypePublic, - a.electionFilterPaginatedHandler, + a.electionListByFilterAndPageHandler, + ); err != nil { + return err + } + if err := a.Endpoint.RegisterMethod( + "/elections/filter", + "POST", + apirest.MethodAccessTypePublic, + a.electionListByFilterHandler, ); err != nil { return err } @@ -124,47 +139,159 @@ func (a *API) enableElectionHandlers() error { return nil } -// electionFullListHandler +// electionListByFilterAndPageHandler // -// @Summary List elections -// @Description Get a list of elections summaries. +// @Summary List elections (filtered) (deprecated, uses url params) +// @Description Deprecated, in favor of /elections/filter // @Tags Elections // @Accept json // @Produce json -// @Param page path number true "Page " -// @Success 200 {object} ElectionSummary -// @Router /elections/page/{page} [get] -func (a *API) electionFullListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - page := 0 - if ctx.URLParam("page") != "" { - var err error - page, err = strconv.Atoi(ctx.URLParam("page")) +// @Param page path number true "Page" +// @Param transaction body ElectionFilter true "Filtered by partial organizationID, partial processID, process status and with results available or not" +// @Success 200 {object} ElectionSummary +// @Router /elections/filter/page/{page} [post] +func (a *API) electionListByFilterAndPageHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { + // get params from the request body + params := &ElectionFilter{} + if err := json.Unmarshal(msg.Data, ¶ms); err != nil { + return ErrCantParseDataAsJSON.WithErr(err) + } + // check that at least one filter is set + if params.OrganizationID == nil && params.ElectionID == nil && params.Status == "" && params.WithResults == nil { + return ErrMissingParameter + } + + if ctx.URLParam(ParamPage) != "" { + urlParams, err := parseElectionFilterParams(ctx.URLParam(ParamPage), "", "", "", "") if err != nil { - return ErrCantParsePageNumber.With(ctx.URLParam("page")) + return err } + params.Page = urlParams.Page + } + return a.sendElectionList(ctx, params) +} + +// electionListByFilterHandler +// +// @Summary List elections (filtered) +// @Description.markdown electionListByFilterHandler +// @Tags Elections +// @Accept json +// @Produce json +// @Param page path number true "Page" +// @Param transaction body ElectionFilter true "Filtered by partial organizationID, partial processID, process status and with results available or not" +// @Success 200 {object} ElectionSummary +// @Router /elections/filter [post] +func (a *API) electionListByFilterHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { + // get params from the request body + params := &ElectionFilter{} + if err := json.Unmarshal(msg.Data, ¶ms); err != nil { + return ErrCantParseDataAsJSON.WithErr(err) } - elections, err := a.indexer.ProcessList(nil, page*MaxPageSize, MaxPageSize, "", 0, 0, "", false) + // check that at least one filter is set + if params.OrganizationID == nil && params.ElectionID == nil && params.Status == "" && params.WithResults == nil { + return ErrMissingParameter + } + + return a.sendElectionList(ctx, params) +} + +// electionListByPageHandler +// +// @Summary List elections (deprecated, uses url params) +// @Description Get a list of elections summaries (Deprecated, in favor of /elections?page=) +// @Tags Elections +// @Accept json +// @Produce json +// @Param page path number true "Page" +// @Success 200 {object} ElectionsList +// @Router /elections/page/{page} [get] +func (a *API) electionListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseElectionFilterParams( + ctx.URLParam(ParamPage), + "", + "", + "", + "", + ) + if err != nil { + return err + } + return a.sendElectionList(ctx, params) +} + +// electionListHandler +// +// @Summary List elections +// @Description Get a list of elections summaries. +// @Tags Elections +// @Accept json +// @Produce json +// @Param page query number false "Page" +// @Param organizationID query string false "Filter by partial organizationID" +// @Param status query string false "Election status" Enums(ready, paused, canceled, ended, results) +// @Param electionID query string false "Filter by partial electionID" +// @Param withResults query boolean false "Return only elections with published results" +// @Success 200 {object} ElectionsList +// @Router /elections [get] +func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + params, err := parseElectionFilterParams( + ctx.QueryParam(ParamPage), + ctx.QueryParam(ParamStatus), + ctx.QueryParam(ParamOrganizationID), + ctx.QueryParam(ParamElectionID), + ctx.QueryParam(ParamWithResults), + ) + if err != nil { + return err + } + return a.sendElectionList(ctx, params) +} + +// sendElectionList produces a filtered, paginated ElectionsList, +// and sends it marshalled over ctx.Send +// +// Errors returned are always of type APIerror. +func (a *API) sendElectionList(ctx *httprouter.HTTPContext, params *ElectionFilter) error { + status, err := parseStatus(params.Status) + if err != nil { + return err + } + + eids, total, err := a.indexer.ProcessList( + params.OrganizationID, + params.Page*ItemsPerPage, + ItemsPerPage, + params.ElectionID.String(), + 0, + 0, + status, + *params.WithResults, + ) if err != nil { return ErrCantFetchElectionList.WithErr(err) } - list := []ElectionSummary{} - for _, eid := range elections { + pagination, err := calculatePagination(params.Page, total) + if err != nil { + return err + } + + if total == 0 { + return ErrElectionNotFound + } + + list := &ElectionsList{ + Pagination: pagination, + } + for _, eid := range eids { e, err := a.indexer.ProcessInfo(eid) if err != nil { return ErrCantFetchElection.Withf("(%x): %v", eid, err) } - list = append(list, a.electionSummary(e)) - } - // wrap list in a struct to consistently return list in an object, return empty - // object if the list does not contains any result - data, err := json.Marshal(struct { - Elections []ElectionSummary `json:"elections"` - }{list}) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) + list.Elections = append(list.Elections, a.electionSummary(e)) } - return ctx.Send(data, apirest.HTTPstatusOK) + return marshalAndSend(ctx, list) } // electionHandler @@ -246,7 +373,7 @@ func (a *API) electionHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) e // @Accept json // @Produce json // @Param electionID path string true "Election id" -// @Success 200 {object} object{count=number} +// @Success 200 {object} CountResult // @Router /elections/{electionID}/votes/count [get] func (a *API) electionVotesCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { electionID, err := hex.DecodeString(util.TrimHex(ctx.URLParam("electionID"))) @@ -265,15 +392,7 @@ func (a *API) electionVotesCountHandler(_ *apirest.APIdata, ctx *httprouter.HTTP } else if err != nil { return ErrCantCountVotes.WithErr(err) } - data, err := json.Marshal( - struct { - Count uint64 `json:"count"` - }{Count: count}, - ) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) + return marshalAndSend(ctx, &CountResult{Count: count}) } // electionKeysHandler @@ -347,16 +466,13 @@ func (a *API) electionVotesHandler(_ *apirest.APIdata, ctx *httprouter.HTTPConte if _, err := getElection(electionID, a.vocapp.State); err != nil { return err } - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber - } + + page, err := parsePage(ctx.URLParam(ParamPage)) + if err != nil { + return err } - page = page * MaxPageSize - votesRaw, err := a.indexer.GetEnvelopes(electionID, MaxPageSize, page, "") + votesRaw, err := a.indexer.GetEnvelopes(electionID, ItemsPerPage, page*ItemsPerPage, "") if err != nil { if errors.Is(err, indexer.ErrVoteNotFound) { return ErrVoteNotFound @@ -612,78 +728,6 @@ func getElection(electionID []byte, vs *state.State) (*models.Process, error) { return process, nil } -// electionFilterPaginatedHandler -// -// @Summary List elections (filtered) -// @Description.markdown electionFilterPaginatedHandler -// @Tags Elections -// @Accept json -// @Produce json -// @Param page path number true "Page to paginate" -// @Param transaction body ElectionFilter true "Filtered by partial organizationID, partial processID, process status and with results available or not" -// @Success 200 {object} ElectionSummary -// @Router /elections/filter/page/{page} [post] -func (a *API) electionFilterPaginatedHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { - // get organizationId from the request body - body := &ElectionFilter{} - if err := json.Unmarshal(msg.Data, &body); err != nil { - return ErrCantParseDataAsJSON.WithErr(err) - } - // check that at least one filter is set - if body.OrganizationID == nil && body.ElectionID == nil && body.Status == "" && body.WithResults == nil { - return ErrMissingParameter - } - // get page - var err error - page := 0 - if ctx.URLParam("page") != "" { - page, err = strconv.Atoi(ctx.URLParam("page")) - if err != nil { - return ErrCantParsePageNumber.WithErr(err) - } - } - page = page * MaxPageSize - if body.WithResults == nil { - withResults := false - body.WithResults = &withResults - } - elections, err := a.indexer.ProcessList( - body.OrganizationID, - page, - MaxPageSize, - body.ElectionID.String(), - 0, - 0, - body.Status, - *body.WithResults, - ) - if err != nil { - return ErrCantFetchElectionList.WithErr(err) - } - if len(elections) == 0 { - return ErrElectionNotFound - } - - var list []ElectionSummary - // get election summary - for _, eid := range elections { - e, err := a.indexer.ProcessInfo(eid) - if err != nil { - return ErrCantFetchElection.WithErr(err) - } - list = append(list, a.electionSummary(e)) - } - data, err := json.Marshal(struct { - Elections []ElectionSummary `json:"elections"` - }{ - Elections: list, - }) - if err != nil { - return ErrMarshalingServerJSONFailed.WithErr(err) - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // buildElectionIDHandler // // @Summary Build an election ID @@ -725,3 +769,34 @@ func (a *API) buildElectionIDHandler(msg *apirest.APIdata, ctx *httprouter.HTTPC } return ctx.Send(data, apirest.HTTPstatusOK) } + +// parseElectionFilterParams returns an ElectionFilter filled with the passed params +func parseElectionFilterParams(page, status, organizationID, electionID, withResults string) (*ElectionFilter, error) { + p, err := parsePage(page) + if err != nil { + return nil, err + } + + oid, err := parseHexString(organizationID) + if err != nil { + return nil, err + } + + eid, err := parseHexString(electionID) + if err != nil { + return nil, err + } + + b, err := parseBool(withResults) + if err != nil { + return nil, err + } + + return &ElectionFilter{ + Page: p, + OrganizationID: oid, + ElectionID: eid, + WithResults: &b, + Status: status, + }, nil +} diff --git a/api/errors.go b/api/errors.go index 6346b0017..090c40faa 100644 --- a/api/errors.go +++ b/api/errors.go @@ -44,7 +44,7 @@ var ( ErrCantParseDataAsJSON = apirest.APIerror{Code: 4016, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse data as JSON")} ErrCantParseElectionID = apirest.APIerror{Code: 4017, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse electionID")} ErrCantParseMetadataAsJSON = apirest.APIerror{Code: 4018, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse metadata (invalid format)")} - ErrCantParsePageNumber = apirest.APIerror{Code: 4019, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse page number")} + ErrCantParseNumber = apirest.APIerror{Code: 4019, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse number")} ErrCantParsePayloadAsJSON = apirest.APIerror{Code: 4020, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse payload as JSON")} ErrCantParseVoteID = apirest.APIerror{Code: 4021, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse voteID")} ErrCantExtractMetadataURI = apirest.APIerror{Code: 4022, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot extract metadata URI")} @@ -56,7 +56,7 @@ var ( ErrCensusTypeMismatch = apirest.APIerror{Code: 4028, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("census type mismatch")} ErrCensusIndexedFlagMismatch = apirest.APIerror{Code: 4029, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("census indexed flag mismatch")} ErrCensusRootHashMismatch = apirest.APIerror{Code: 4030, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("census root hash mismatch after importing dump")} - ErrParamStatusMissing = apirest.APIerror{Code: 4031, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("parameter (status) missing or invalid")} + ErrParamStatusInvalid = apirest.APIerror{Code: 4031, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("parameter (status) invalid")} ErrParamParticipantsMissing = apirest.APIerror{Code: 4032, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("parameter (participants) missing")} ErrParamParticipantsTooBig = apirest.APIerror{Code: 4033, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("parameter (participants) exceeds max length per call")} ErrParamDumpOrRootMissing = apirest.APIerror{Code: 4034, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("parameter (dump or root) missing")} @@ -80,6 +80,9 @@ var ( ErrUnmarshalingServerProto = apirest.APIerror{Code: 4052, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error unmarshaling protobuf data")} ErrMarshalingServerProto = apirest.APIerror{Code: 4053, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error marshaling protobuf data")} ErrSIKNotFound = apirest.APIerror{Code: 4054, HTTPstatus: apirest.HTTPstatusNotFound, Err: fmt.Errorf("SIK not found")} + ErrCantParseBoolean = apirest.APIerror{Code: 4055, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse string into boolean")} + ErrCantParseHexString = apirest.APIerror{Code: 4056, HTTPstatus: apirest.HTTPstatusBadRequest, Err: fmt.Errorf("cannot parse string into hex bytes")} + ErrPageNotFound = apirest.APIerror{Code: 4057, HTTPstatus: apirest.HTTPstatusNotFound, Err: fmt.Errorf("page not found")} ErrVochainEmptyReply = apirest.APIerror{Code: 5000, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("vochain returned an empty reply")} ErrVochainSendTxFailed = apirest.APIerror{Code: 5001, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("vochain SendTx failed")} ErrVochainGetTxFailed = apirest.APIerror{Code: 5002, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("vochain GetTx failed")} @@ -113,4 +116,5 @@ var ( ErrVochainOverloaded = apirest.APIerror{Code: 5030, HTTPstatus: apirest.HTTPstatusServiceUnavailable, Err: fmt.Errorf("vochain overloaded")} ErrGettingSIK = apirest.APIerror{Code: 5031, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error getting SIK")} ErrCensusBuild = apirest.APIerror{Code: 5032, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error building census")} + ErrIndexerQueryFailed = apirest.APIerror{Code: 5033, HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("indexer query failed")} ) diff --git a/api/helpers.go b/api/helpers.go index 813ba926e..e9e43f5ea 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -5,7 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" + "strconv" + "strings" cometpool "github.com/cometbft/cometbft/mempool" cometcoretypes "github.com/cometbft/cometbft/rpc/core/types" @@ -13,7 +16,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/iancoleman/strcase" "go.vocdoni.io/dvote/crypto/nacl" + "go.vocdoni.io/dvote/httprouter" + "go.vocdoni.io/dvote/httprouter/apirest" "go.vocdoni.io/dvote/types" + "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/proto/build/go/models" "google.golang.org/protobuf/encoding/protojson" @@ -162,3 +168,118 @@ func decryptVotePackage(vp []byte, privKeys []string, indexes []uint32) ([]byte, } return vp, nil } + +// marshalAndSend marshals any passed struct and sends it over ctx.Send() +func marshalAndSend(ctx *httprouter.HTTPContext, v any) error { + data, err := json.Marshal(v) + if err != nil { + return ErrMarshalingServerJSONFailed.WithErr(err) + } + return ctx.Send(data, apirest.HTTPstatusOK) +} + +// parseNumber parses a string into an int. +// +// If the string is not parseable, returns an APIerror. +// +// The empty string "" is treated specially, returns 0 with no error. +func parseNumber(s string) (int, error) { + if s == "" { + return 0, nil + } + page, err := strconv.Atoi(s) + if err != nil { + return 0, ErrCantParseNumber.With(s) + } + return page, nil +} + +// parsePage parses a string into an int. +// +// If the resulting int is negative, returns ErrNoSuchPage. +// If the string is not parseable, returns an APIerror. +// +// The empty string "" is treated specially, returns 0 with no error. +func parsePage(s string) (int, error) { + page, err := parseNumber(s) + if err != nil { + return 0, err + } + if page < 0 { + return 0, ErrPageNotFound + } + return page, nil +} + +// parseStatus converts a string ("READY", "ready", "PAUSED", etc) +// to a models.ProcessStatus. +// +// If the string doesn't map to a value, returns an APIerror. +// +// The empty string "" is treated specially, returns 0 with no error. +func parseStatus(s string) (models.ProcessStatus, error) { + if s == "" { + return 0, nil + } + status, found := models.ProcessStatus_value[strings.ToUpper(s)] + if !found { + return 0, ErrParamStatusInvalid.With(s) + } + return models.ProcessStatus(status), nil +} + +// parseHexString converts a string like 0x1234cafe (or 1234cafe) +// to a types.HexBytes. +// +// If the string can't be parsed, returns an APIerror. +func parseHexString(s string) (types.HexBytes, error) { + orgID, err := hex.DecodeString(util.TrimHex(s)) + if err != nil { + return nil, ErrCantParseHexString.Withf("%q", s) + } + return orgID, nil +} + +// parseBool parses a string into a boolean value. +// +// The empty string "" is treated specially, returns false with no error. +func parseBool(s string) (bool, error) { + if s == "" { + return false, nil + } + b, err := strconv.ParseBool(s) + if err != nil { + return false, ErrCantParseBoolean.With(s) + } + return b, nil +} + +// calculatePagination calculates PreviousPage, NextPage and LastPage. +// +// If page is negative or higher than LastPage, returns an APIerror (ErrPageNotFound) +func calculatePagination(page int, totalItems uint64) (*Pagination, error) { + // pages start at 0 index, for legacy reasons + lastp := int(math.Ceil(float64(totalItems)/ItemsPerPage) - 1) + + if page > lastp || page < 0 { + return nil, ErrPageNotFound + } + + var prevp, nextp *uint64 + if page > 0 { + prevPage := uint64(page - 1) + prevp = &prevPage + } + if page < lastp { + nextPage := uint64(page + 1) + nextp = &nextPage + } + + return &Pagination{ + TotalItems: totalItems, + PreviousPage: prevp, + CurrentPage: uint64(page), + NextPage: nextp, + LastPage: uint64(lastp), + }, nil +} diff --git a/httprouter/message.go b/httprouter/message.go index c17a12fcd..b1723dd29 100644 --- a/httprouter/message.go +++ b/httprouter/message.go @@ -44,6 +44,12 @@ func (h *HTTPContext) URLParam(key string) string { return chi.URLParam(h.Request, key) } +// QueryParam is a wrapper around go-chi to get the value of a query string parameter (like "?key=value"). +// If key is not present, returns the empty string. +func (h *HTTPContext) QueryParam(key string) string { + return h.Request.URL.Query().Get(key) +} + // Send replies the request with the provided message. func (h *HTTPContext) Send(msg []byte, httpStatusCode int) error { defer func() { diff --git a/test/api_test.go b/test/api_test.go index 3308d1be8..f9e433976 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -87,7 +87,7 @@ func TestAPIcensusAndVote(t *testing.T) { qt.Assert(t, censusData.Weight.String(), qt.Equals, "1") electionParams := electionprice.ElectionParameters{ElectionDuration: 100, MaxCensusSize: 100} - election := createElection(t, c, server.Account, electionParams, censusData.CensusRoot, 0, server.VochainAPP.ChainID(), false) + election := createElection(t, c, server.Account, electionParams, censusData.CensusRoot, 0, server.VochainAPP.ChainID(), false, 0) // Block 2 server.VochainAPP.AdvanceTestBlock() @@ -216,6 +216,63 @@ func TestAPIaccount(t *testing.T) { qt.Assert(t, gotAcct.Balance, qt.Equals, initBalance) } +func TestAPIAccountsList(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 1) + + // create new accounts + for nonce := uint32(0); nonce < 20; nonce++ { + createAccount(t, c, server, uint64(80)) + } + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + // Get the list and check it + fetchAL := func(method string, jsonBody any, query string, urlPath ...string) api.AccountsList { + resp, code := c.RequestWithQuery(method, jsonBody, query, urlPath...) + list := api.AccountsList{} + qt.Assert(t, code, qt.Equals, 200) + err := json.Unmarshal(resp, &list) + qt.Assert(t, err, qt.IsNil) + return list + } + + el := make(map[string]api.AccountsList) + el["0"] = fetchAL("GET", nil, "", "accounts") + el["1"] = fetchAL("GET", nil, "page=1", "accounts") + el["p0"] = fetchAL("GET", nil, "", "accounts", "page", "0") + el["p1"] = fetchAL("GET", nil, "", "accounts", "page", "1") + + qt.Assert(t, el["0"], qt.Not(qt.DeepEquals), el["1"]) + qt.Assert(t, el["0"], qt.DeepEquals, el["p0"]) + qt.Assert(t, el["1"], qt.DeepEquals, el["p1"]) + + // 2 accounts pre-exist: the faucet account, and the burn address + qt.Assert(t, el["0"].Pagination.TotalItems, qt.Equals, uint64(2+20)) + qt.Assert(t, el["1"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + qt.Assert(t, el["p0"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + qt.Assert(t, el["p1"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + + for _, item := range el { + qt.Assert(t, len(item.Accounts), qt.Equals, api.ItemsPerPage) + } +} + func TestAPIElectionCost(t *testing.T) { // cheap election runAPIElectionCostWithParams(t, @@ -442,7 +499,7 @@ func runAPIElectionCostWithParams(t *testing.T, qt.Assert(t, requestAccount(t, c, signer.Address().String()).Balance, qt.Equals, initialBalance) - createElection(t, c, signer, electionParams, censusRoot, startBlock, server.VochainAPP.ChainID(), false) + createElection(t, c, signer, electionParams, censusRoot, startBlock, server.VochainAPP.ChainID(), false, 0) // Block 3 server.VochainAPP.AdvanceTestBlock() @@ -508,6 +565,7 @@ func createElection(t testing.TB, c *testutil.TestHTTPclient, startBlock uint32, chainID string, encryptedMetadata bool, + nonce uint32, ) api.ElectionCreate { metadataBytes, err := json.Marshal( &api.ElectionMetadata{ @@ -530,7 +588,7 @@ func createElection(t testing.TB, c *testutil.TestHTTPclient, tx := models.Tx_NewProcess{ NewProcess: &models.NewProcessTx{ Txtype: models.TxType_NEW_PROCESS, - Nonce: 0, + Nonce: nonce, Process: &models.Process{ StartBlock: startBlock, BlockCount: electionParams.ElectionDuration, @@ -737,7 +795,7 @@ func TestAPIBuildElectionID(t *testing.T) { // create a new election electionParams := electionprice.ElectionParameters{ElectionDuration: 100, MaxCensusSize: 100} - response := createElection(t, c, signer, electionParams, censusRoot, 0, server.VochainAPP.ChainID(), false) + response := createElection(t, c, signer, electionParams, censusRoot, 0, server.VochainAPP.ChainID(), false, 0) // Block 4 server.VochainAPP.AdvanceTestBlock() @@ -807,7 +865,7 @@ func TestAPIEncryptedMetadata(t *testing.T) { // create a new election electionParams := electionprice.ElectionParameters{ElectionDuration: 100, MaxCensusSize: 100} - electionResponse := createElection(t, c, signer, electionParams, censusRoot, 0, server.VochainAPP.ChainID(), true) + electionResponse := createElection(t, c, signer, electionParams, censusRoot, 0, server.VochainAPP.ChainID(), true, 0) // Block 4 server.VochainAPP.AdvanceTestBlock() @@ -833,3 +891,64 @@ func TestAPIEncryptedMetadata(t *testing.T) { qt.Assert(t, err, qt.IsNil) qt.Assert(t, metadata.Title["default"], qt.Equals, "test election") } + +func TestAPIElectionsList(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + // Block 1 + server.VochainAPP.AdvanceTestBlock() + + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // create a new census + resp, code := c.Request("POST", nil, "censuses", "weighted") + qt.Assert(t, code, qt.Equals, 200) + censusData := &api.Census{} + qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil) + + electionParams := electionprice.ElectionParameters{ElectionDuration: 100, MaxCensusSize: 100} + for nonce := uint32(0); nonce < 20; nonce++ { + createElection(t, c, server.Account, electionParams, censusData.CensusRoot, 0, server.VochainAPP.ChainID(), false, nonce) + } + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + // Get the list of elections and check it + fetchEL := func(method string, jsonBody any, query string, urlPath ...string) api.ElectionsList { + resp, code := c.RequestWithQuery(method, jsonBody, query, urlPath...) + elections := api.ElectionsList{} + qt.Assert(t, code, qt.Equals, 200) + err := json.Unmarshal(resp, &elections) + qt.Assert(t, err, qt.IsNil) + return elections + } + + el := make(map[string]api.ElectionsList) + el["0"] = fetchEL("GET", nil, "", "elections") + el["1"] = fetchEL("GET", nil, "page=1", "elections") + el["p0"] = fetchEL("GET", nil, "", "elections", "page", "0") + el["p1"] = fetchEL("GET", nil, "", "elections", "page", "1") + + qt.Assert(t, el["0"], qt.Not(qt.DeepEquals), el["1"]) + qt.Assert(t, el["0"], qt.DeepEquals, el["p0"]) + qt.Assert(t, el["1"], qt.DeepEquals, el["p1"]) + + qt.Assert(t, el["0"].Pagination.TotalItems, qt.Equals, uint64(20)) + qt.Assert(t, el["1"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + qt.Assert(t, el["p0"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + qt.Assert(t, el["p1"].Pagination.TotalItems, qt.Equals, el["0"].Pagination.TotalItems) + + for _, item := range el { + qt.Assert(t, len(item.Elections), qt.Equals, api.ItemsPerPage) + } +} diff --git a/test/apierror_test.go b/test/apierror_test.go index 6cb688b5f..9beeafd90 100644 --- a/test/apierror_test.go +++ b/test/apierror_test.go @@ -65,7 +65,7 @@ func TestAPIerror(t *testing.T) { }, { args: args{"GET", nil, []string{"accounts", "totallyWrong!@#$", "elections", "status", "ready", "page", "0"}}, - want: api.ErrCantParseOrgID, + want: api.ErrCantParseHexString, }, { args: args{"GET", nil, []string{"accounts", "totallyWrong!@#$", "transfers", "page", "0"}}, @@ -110,11 +110,19 @@ func TestAPIerror(t *testing.T) { "status", "ready", "page", "-1", }}, - want: api.ErrCantFetchElectionList, + want: api.ErrPageNotFound, }, { args: args{"GET", nil, []string{"elections", "page", "thisIsTotallyNotAnInt"}}, - want: api.ErrCantParsePageNumber, + want: api.ErrCantParseNumber, + }, + { + args: args{"GET", nil, []string{"elections", "page", "1"}}, + want: api.ErrPageNotFound, + }, + { + args: args{"GET", nil, []string{"elections", "page", "-1"}}, + want: api.ErrPageNotFound, }, } for _, tt := range tests { diff --git a/test/testcommon/testutil/apiclient.go b/test/testcommon/testutil/apiclient.go index d7ae50a2f..7a560dbf7 100644 --- a/test/testcommon/testutil/apiclient.go +++ b/test/testcommon/testutil/apiclient.go @@ -21,16 +21,27 @@ type TestHTTPclient struct { t testing.TB } -func (c *TestHTTPclient) Request(method string, jsonBody any, urlPath ...string) ([]byte, int) { - body, err := json.Marshal(jsonBody) +func (c *TestHTTPclient) RequestWithQuery(method string, jsonBody any, query string, urlPath ...string) ([]byte, int) { + u, err := url.Parse(c.addr.String()) qt.Assert(c.t, err, qt.IsNil) + u.RawQuery = query + return c.request(method, u, jsonBody, urlPath...) +} + +func (c *TestHTTPclient) Request(method string, jsonBody any, urlPath ...string) ([]byte, int) { u, err := url.Parse(c.addr.String()) qt.Assert(c.t, err, qt.IsNil) + return c.request(method, u, jsonBody, urlPath...) +} + +func (c *TestHTTPclient) request(method string, u *url.URL, jsonBody any, urlPath ...string) ([]byte, int) { u.Path = path.Join(u.Path, path.Join(urlPath...)) headers := http.Header{} if c.token != nil { headers = http.Header{"Authorization": []string{"Bearer " + c.token.String()}} } + body, err := json.Marshal(jsonBody) + qt.Assert(c.t, err, qt.IsNil) c.t.Logf("querying %s", u) resp, err := c.c.Do(&http.Request{ Method: method, diff --git a/vochain/indexer/db/account.sql.go b/vochain/indexer/db/account.sql.go index c34ebc307..6e1691e5d 100644 --- a/vochain/indexer/db/account.sql.go +++ b/vochain/indexer/db/account.sql.go @@ -13,8 +13,6 @@ import ( ) const countAccounts = `-- name: CountAccounts :one -; - SELECT COUNT(*) FROM accounts ` @@ -42,9 +40,8 @@ func (q *Queries) CreateAccount(ctx context.Context, arg CreateAccountParams) (s } const getListAccounts = `-- name: GetListAccounts :many -; - -SELECT account, balance, nonce +SELECT account, balance, nonce, + COUNT(*) OVER() AS total_count FROM accounts ORDER BY balance DESC LIMIT ? OFFSET ? @@ -55,16 +52,28 @@ type GetListAccountsParams struct { Offset int64 } -func (q *Queries) GetListAccounts(ctx context.Context, arg GetListAccountsParams) ([]Account, error) { +type GetListAccountsRow struct { + Account types.AccountID + Balance int64 + Nonce int64 + TotalCount int64 +} + +func (q *Queries) GetListAccounts(ctx context.Context, arg GetListAccountsParams) ([]GetListAccountsRow, error) { rows, err := q.query(ctx, q.getListAccountsStmt, getListAccounts, arg.Limit, arg.Offset) if err != nil { return nil, err } defer rows.Close() - var items []Account + var items []GetListAccountsRow for rows.Next() { - var i Account - if err := rows.Scan(&i.Account, &i.Balance, &i.Nonce); err != nil { + var i GetListAccountsRow + if err := rows.Scan( + &i.Account, + &i.Balance, + &i.Nonce, + &i.TotalCount, + ); err != nil { return nil, err } items = append(items, i) diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index a1566a32a..466cf1a8f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -10,12 +10,6 @@ import ( "go.vocdoni.io/dvote/types" ) -type Account struct { - Account types.AccountID - Balance int64 - Nonce int64 -} - type Block struct { Height int64 Time time.Time diff --git a/vochain/indexer/db/processes.sql.go b/vochain/indexer/db/processes.sql.go index 73220d6b3..d15ffe39b 100644 --- a/vochain/indexer/db/processes.sql.go +++ b/vochain/indexer/db/processes.sql.go @@ -176,8 +176,6 @@ func (q *Queries) GetProcessCount(ctx context.Context) (int64, error) { } const getProcessIDsByFinalResults = `-- name: GetProcessIDsByFinalResults :many -; - SELECT id FROM processes WHERE final_results = ? ` @@ -219,27 +217,34 @@ func (q *Queries) GetProcessStatus(ctx context.Context, id types.ProcessID) (int } const searchEntities = `-- name: SearchEntities :many -SELECT entity_id, COUNT(id) AS process_count FROM processes -WHERE (?1 = '' OR (INSTR(LOWER(HEX(entity_id)), ?1) > 0)) +WITH results AS ( + SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, + COUNT(*) OVER() AS total_count + FROM processes + WHERE (?3 = '' OR (INSTR(LOWER(HEX(entity_id)), ?3) > 0)) +) +SELECT entity_id, COUNT(id) AS process_count, total_count +FROM results GROUP BY entity_id ORDER BY creation_time DESC, id ASC -LIMIT ?3 -OFFSET ?2 +LIMIT ?2 +OFFSET ?1 ` type SearchEntitiesParams struct { - EntityIDSubstr interface{} Offset int64 Limit int64 + EntityIDSubstr interface{} } type SearchEntitiesRow struct { - EntityID types.EntityID + EntityID []byte ProcessCount int64 + TotalCount int64 } func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) ([]SearchEntitiesRow, error) { - rows, err := q.query(ctx, q.searchEntitiesStmt, searchEntities, arg.EntityIDSubstr, arg.Offset, arg.Limit) + rows, err := q.query(ctx, q.searchEntitiesStmt, searchEntities, arg.Offset, arg.Limit, arg.EntityIDSubstr) if err != nil { return nil, err } @@ -247,7 +252,7 @@ func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) var items []SearchEntitiesRow for rows.Next() { var i SearchEntitiesRow - if err := rows.Scan(&i.EntityID, &i.ProcessCount); err != nil { + if err := rows.Scan(&i.EntityID, &i.ProcessCount, &i.TotalCount); err != nil { return nil, err } items = append(items, i) @@ -262,52 +267,63 @@ func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) } const searchProcesses = `-- name: SearchProcesses :many -SELECT id FROM processes -WHERE (LENGTH(?1) = 0 OR entity_id = ?1) - AND (?2 = 0 OR namespace = ?2) - AND (?3 = 0 OR status = ?3) - AND (?4 = 0 OR source_network_id = ?4) - -- TODO(mvdan): consider keeping an id_hex column for faster searches - AND (?5 = '' OR (INSTR(LOWER(HEX(id)), ?5) > 0)) - AND (?6 = FALSE OR have_results) +WITH results AS ( + SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, + COUNT(*) OVER() AS total_count + FROM processes + WHERE (LENGTH(?3) = 0 OR entity_id = ?3) + AND (?4 = 0 OR namespace = ?4) + AND (?5 = 0 OR status = ?5) + AND (?6 = 0 OR source_network_id = ?6) + -- TODO: consider keeping an id_hex column for faster searches + AND (?7 = '' OR (INSTR(LOWER(HEX(id)), ?7) > 0)) + AND (?8 = FALSE OR have_results) +) +SELECT id, total_count +FROM results ORDER BY creation_time DESC, id ASC -LIMIT ?8 -OFFSET ?7 +LIMIT ?2 +OFFSET ?1 ` type SearchProcessesParams struct { + Offset int64 + Limit int64 EntityID interface{} Namespace interface{} Status interface{} SourceNetworkID interface{} IDSubstr interface{} WithResults interface{} - Offset int64 - Limit int64 } -func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]types.ProcessID, error) { +type SearchProcessesRow struct { + ID []byte + TotalCount int64 +} + +func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]SearchProcessesRow, error) { rows, err := q.query(ctx, q.searchProcessesStmt, searchProcesses, + arg.Offset, + arg.Limit, arg.EntityID, arg.Namespace, arg.Status, arg.SourceNetworkID, arg.IDSubstr, arg.WithResults, - arg.Offset, - arg.Limit, ) if err != nil { return nil, err } defer rows.Close() - var items []types.ProcessID + var items []SearchProcessesRow for rows.Next() { - var id types.ProcessID - if err := rows.Scan(&id); err != nil { + var i SearchProcessesRow + if err := rows.Scan(&i.ID, &i.TotalCount); err != nil { return nil, err } - items = append(items, id) + items = append(items, i) } if err := rows.Close(); err != nil { return nil, err @@ -382,8 +398,6 @@ func (q *Queries) UpdateProcessEndDate(ctx context.Context, arg UpdateProcessEnd } const updateProcessFromState = `-- name: UpdateProcessFromState :execresult -; - UPDATE processes SET census_root = ?1, census_uri = ?2, diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index c0b6d0289..11fa1d4bf 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -905,21 +905,24 @@ func (idx *Indexer) CountTotalAccounts() (uint64, error) { return uint64(count), err } -func (idx *Indexer) GetListAccounts(offset, maxItems int32) ([]indexertypes.Account, error) { - accsFromDB, err := idx.readOnlyQuery.GetListAccounts(context.TODO(), indexerdb.GetListAccountsParams{ +func (idx *Indexer) AccountsList(offset, maxItems int) ([]indexertypes.Account, uint64, error) { + results, err := idx.readOnlyQuery.GetListAccounts(context.TODO(), indexerdb.GetListAccountsParams{ Limit: int64(maxItems), Offset: int64(offset), }) if err != nil { - return nil, err + return nil, 0, err + } + if len(results) == 0 { + return []indexertypes.Account{}, 0, nil } - tt := []indexertypes.Account{} - for _, acc := range accsFromDB { - tt = append(tt, indexertypes.Account{ - Address: acc.Account, - Balance: uint64(acc.Balance), - Nonce: uint32(acc.Nonce), + list := []indexertypes.Account{} + for _, row := range results { + list = append(list, indexertypes.Account{ + Address: row.Account, + Balance: uint64(row.Balance), + Nonce: uint32(row.Nonce), }) } - return tt, nil + return list, uint64(results[0].TotalCount), nil } diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index ce0b8c9c7..eee575bfa 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -160,7 +160,8 @@ func testEntityList(t *testing.T, entityCount int) { entitiesByID := make(map[string]bool) last := 0 for len(entitiesByID) <= entityCount { - list := idx.EntityList(10, last, "") + list, _, err := idx.EntityList(last, 10, "") + qt.Assert(t, err, qt.IsNil) if len(list) < 1 { t.Log("list is empty") break @@ -254,17 +255,20 @@ func TestEntitySearch(t *testing.T) { } app.AdvanceTestBlock() // Exact entity search - list := idx.EntityList(10, 0, "4011d50537fa164b6fef261141797bbe4014526e") + list, _, err := idx.EntityList(0, 10, "4011d50537fa164b6fef261141797bbe4014526e") + qt.Assert(t, err, qt.IsNil) if len(list) < 1 { t.Fatalf("expected 1 entity, got %d", len(list)) } // Search for nonexistent entity - list = idx.EntityList(10, 0, "4011d50537fa164b6fef261141797bbe4014526f") + list, _, err = idx.EntityList(0, 10, "4011d50537fa164b6fef261141797bbe4014526f") + qt.Assert(t, err, qt.IsNil) if len(list) > 0 { t.Fatalf("expected 0 entities, got %d", len(list)) } // Search containing part of all manually-defined entities - list = idx.EntityList(10, 0, "011d50537fa164b6fef261141797bbe4014526e") + list, _, err = idx.EntityList(0, 10, "011d50537fa164b6fef261141797bbe4014526e") + qt.Assert(t, err, qt.IsNil) log.Info(list) if len(list) < len(entityIds) { t.Fatalf("expected %d entities, got %d", len(entityIds), len(list)) @@ -324,10 +328,11 @@ func testProcessList(t *testing.T, procsCount int) { procs := make(map[string]bool) last := 0 for len(procs) < procsCount { - list, err := idx.ProcessList(eidProcsCount, last, 10, "", 0, 0, "", false) + list, total, err := idx.ProcessList(eidProcsCount, last, 10, "", 0, 0, 0, false) if err != nil { t.Fatal(err) } + qt.Assert(t, total, qt.Equals, uint64(procsCount)) if len(list) < 1 { t.Log("list is empty") break @@ -342,12 +347,14 @@ func testProcessList(t *testing.T, procsCount int) { } qt.Assert(t, procs, qt.HasLen, procsCount) - _, err := idx.ProcessList(nil, 0, 64, "", 0, 0, "", false) + _, total, err := idx.ProcessList(nil, 0, 64, "", 0, 0, 0, false) qt.Assert(t, err, qt.IsNil) + qt.Assert(t, total, qt.Equals, uint64(10+procsCount)) qt.Assert(t, idx.CountTotalProcesses(), qt.Equals, uint64(10+procsCount)) countEntityProcs := func(eid []byte) int64 { - list := idx.EntityList(1, 0, fmt.Sprintf("%x", eid)) + list, _, err := idx.EntityList(0, 1, fmt.Sprintf("%x", eid)) + qt.Assert(t, err, qt.IsNil) if len(list) == 0 { return -1 } @@ -356,6 +363,11 @@ func testProcessList(t *testing.T, procsCount int) { qt.Assert(t, countEntityProcs(eidOneProcess), qt.Equals, int64(1)) qt.Assert(t, countEntityProcs(eidProcsCount), qt.Equals, int64(procsCount)) qt.Assert(t, countEntityProcs([]byte("not an entity id that exists")), qt.Equals, int64(-1)) + + // Past the end (from=10000) should return an empty list + emptyList, _, err := idx.ProcessList(nil, 10000, 64, "", 0, 0, 0, false) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, emptyList, qt.DeepEquals, [][]byte{}) } func TestProcessSearch(t *testing.T) { @@ -443,7 +455,7 @@ func TestProcessSearch(t *testing.T) { app.AdvanceTestBlock() // Exact process search - list, err := idx.ProcessList(eidTest, 0, 10, pidExact, 0, 0, "", false) + list, _, err := idx.ProcessList(eidTest, 0, 10, pidExact, 0, 0, 0, false) if err != nil { t.Fatal(err) } @@ -452,7 +464,7 @@ func TestProcessSearch(t *testing.T) { } // Exact process search, with it being encrypted. // This once caused a sqlite bug due to a mistake in the SQL query. - list, err = idx.ProcessList(eidTest, 0, 10, pidExactEncrypted, 0, 0, "", false) + list, _, err = idx.ProcessList(eidTest, 0, 10, pidExactEncrypted, 0, 0, 0, false) if err != nil { t.Fatal(err) } @@ -460,8 +472,8 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected 1 process, got %d", len(list)) } // Search for nonexistent process - list, err = idx.ProcessList(eidTest, 0, 10, - "4011d50537fa164b6fef261141797bbe4014526f", 0, 0, "", false) + list, _, err = idx.ProcessList(eidTest, 0, 10, + "4011d50537fa164b6fef261141797bbe4014526f", 0, 0, 0, false) if err != nil { t.Fatal(err) } @@ -469,8 +481,8 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected 0 processes, got %d", len(list)) } // Search containing part of all manually-defined processes - list, err = idx.ProcessList(eidTest, 0, 10, - "011d50537fa164b6fef261141797bbe4014526e", 0, 0, "", false) + list, _, err = idx.ProcessList(eidTest, 0, 10, + "011d50537fa164b6fef261141797bbe4014526e", 0, 0, 0, false) if err != nil { t.Fatal(err) } @@ -478,8 +490,8 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected %d processes, got %d", len(processIds), len(list)) } - list, err = idx.ProcessList(eidTest, 0, 100, - "0c6ca22d2c175a1fbdd15d7595ae532bb1094b5", 0, 0, "ENDED", false) + list, _, err = idx.ProcessList(eidTest, 0, 100, + "0c6ca22d2c175a1fbdd15d7595ae532bb1094b5", 0, 0, models.ProcessStatus_ENDED, false) if err != nil { t.Fatal(err) } @@ -489,7 +501,7 @@ func TestProcessSearch(t *testing.T) { // Search with an exact Entity ID, but starting with a null byte. // This can trip up sqlite, as it assumes TEXT strings are NUL-terminated. - list, err = idx.ProcessList([]byte("\x00foobar"), 0, 100, "", 0, 0, "", false) + list, _, err = idx.ProcessList([]byte("\x00foobar"), 0, 100, "", 0, 0, 0, false) if err != nil { t.Fatal(err) } @@ -498,12 +510,12 @@ func TestProcessSearch(t *testing.T) { } // list all processes, with a max of 10 - list, err = idx.ProcessList(nil, 0, 10, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 10, "", 0, 0, 0, false) qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 10) // list all processes, with a max of 1000 - list, err = idx.ProcessList(nil, 0, 1000, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 1000, "", 0, 0, 0, false) qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 21) } @@ -552,25 +564,25 @@ func TestProcessListWithNamespaceAndStatus(t *testing.T) { app.AdvanceTestBlock() // Get the process list for namespace 123 - list, err := idx.ProcessList(eid20, 0, 100, "", 123, 0, "", false) + list, _, err := idx.ProcessList(eid20, 0, 100, "", 123, 0, 0, false) qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 qt.Assert(t, len(list), qt.CmpEquals(), 10) // Get the process list for all namespaces - list, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 0, 0, 0, false) qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 + 10 qt.Assert(t, len(list), qt.CmpEquals(), 20) // Get the process list for namespace 10 - list, err = idx.ProcessList(nil, 0, 100, "", 10, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 10, 0, 0, false) qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 1) // Get the process list for namespace 10 - list, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "READY", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 0, 0, models.ProcessStatus_READY, false) qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 10) @@ -1522,7 +1534,7 @@ func TestAccountsList(t *testing.T) { last := 0 for i := 0; i < int(totalAccs); i++ { - accts, err := idx.GetListAccounts(int32(last), 10) + accts, _, err := idx.AccountsList(last, 10) qt.Assert(t, err, qt.IsNil) for j, acc := range accts { @@ -1545,7 +1557,7 @@ func TestAccountsList(t *testing.T) { app.AdvanceTestBlock() // verify the updated balance and nonce - accts, err := idx.GetListAccounts(int32(0), 5) + accts, _, err := idx.AccountsList(0, 5) qt.Assert(t, err, qt.IsNil) // the account in the position 0 must be the updated account balance due it has the major balance // indexer query has order BY balance DESC diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 31ba97d73..1ce755c16 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -236,3 +236,8 @@ type TokenTransfersAccount struct { Received []*TokenTransferMeta `json:"received"` Sent []*TokenTransferMeta `json:"sent"` } + +type Entity struct { + EntityID types.EntityID + ProcessCount int64 +} diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index 3baeb08f4..c768aa2f3 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -48,29 +48,19 @@ func (idx *Indexer) ProcessInfo(pid []byte) (*indexertypes.Process, error) { // declared as zero-values will be ignored. SearchTerm is a partial or full PID. // Status is one of READY, CANCELED, ENDED, PAUSED, RESULTS func (idx *Indexer) ProcessList(entityID []byte, from, max int, searchTerm string, namespace uint32, - srcNetworkId int32, status string, withResults bool, -) ([][]byte, error) { + srcNetworkId int32, status models.ProcessStatus, withResults bool, +) ([][]byte, uint64, error) { if from < 0 { - return nil, fmt.Errorf("processList: invalid value: from is invalid value %d", from) - } - // For filtering on Status we use a badgerhold match function. - // If status is not defined, then the match function will return always true. - statusnum := int32(0) - statusfound := false - if status != "" { - if statusnum, statusfound = models.ProcessStatus_value[status]; !statusfound { - return nil, fmt.Errorf("processList: status %s is unknown", status) - } + return nil, 0, fmt.Errorf("processList: invalid value: from is invalid value %d", from) } // Filter match function for source network Id if _, ok := models.SourceNetworkId_name[srcNetworkId]; !ok { - return nil, fmt.Errorf("sourceNetworkId is unknown %d", srcNetworkId) + return nil, 0, fmt.Errorf("sourceNetworkId is unknown %d", srcNetworkId) } - - procs, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ + results, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ EntityID: nonNullBytes(entityID), // so that LENGTH never returns NULL Namespace: int64(namespace), - Status: int64(statusnum), + Status: int64(status), SourceNetworkID: int64(srcNetworkId), IDSubstr: searchTerm, Offset: int64(from), @@ -78,9 +68,16 @@ func (idx *Indexer) ProcessList(entityID []byte, from, max int, searchTerm strin WithResults: withResults, }) if err != nil { - return nil, err + return nil, 0, err + } + if len(results) == 0 { + return [][]byte{}, 0, nil } - return procs, nil + list := [][]byte{} + for _, row := range results { + list = append(list, row.ID) + } + return list, uint64(results[0].TotalCount), nil } // CountTotalProcesses returns the total number of processes indexed. @@ -96,17 +93,26 @@ func (idx *Indexer) CountTotalProcesses() uint64 { // EntityList returns the list of entities indexed by the indexer // searchTerm is optional, if declared as zero-value // will be ignored. Searches against the ID field. -func (idx *Indexer) EntityList(max, from int, searchTerm string) []indexerdb.SearchEntitiesRow { - rows, err := idx.readOnlyQuery.SearchEntities(context.TODO(), indexerdb.SearchEntitiesParams{ +func (idx *Indexer) EntityList(from, max int, searchTerm string) ([]indexertypes.Entity, uint64, error) { + results, err := idx.readOnlyQuery.SearchEntities(context.TODO(), indexerdb.SearchEntitiesParams{ EntityIDSubstr: searchTerm, Offset: int64(from), Limit: int64(max), }) if err != nil { - log.Errorf("error listing entities: %v", err) - return nil + return nil, 0, fmt.Errorf("error listing entities: %w", err) + } + if len(results) == 0 { + return nil, 0, nil + } + list := []indexertypes.Entity{} + for _, row := range results { + list = append(list, indexertypes.Entity{ + EntityID: row.EntityID, + ProcessCount: row.ProcessCount, + }) } - return rows + return list, uint64(results[0].TotalCount), nil } // CountTotalEntities return the total number of entities indexed by the indexer diff --git a/vochain/indexer/queries/account.sql b/vochain/indexer/queries/account.sql index e459ff375..630e14657 100644 --- a/vochain/indexer/queries/account.sql +++ b/vochain/indexer/queries/account.sql @@ -1,15 +1,14 @@ -- name: CreateAccount :execresult REPLACE INTO accounts ( account, balance, nonce -) VALUES (?, ?, ?) -; +) VALUES (?, ?, ?); -- name: GetListAccounts :many -SELECT * +SELECT *, + COUNT(*) OVER() AS total_count FROM accounts ORDER BY balance DESC -LIMIT ? OFFSET ? -; +LIMIT ? OFFSET ?; -- name: CountAccounts :one SELECT COUNT(*) FROM accounts; \ No newline at end of file diff --git a/vochain/indexer/queries/processes.sql b/vochain/indexer/queries/processes.sql index 919eeccab..bc57edce4 100644 --- a/vochain/indexer/queries/processes.sql +++ b/vochain/indexer/queries/processes.sql @@ -31,18 +31,23 @@ WHERE id = ? LIMIT 1; -- name: SearchProcesses :many -SELECT id FROM processes -WHERE (LENGTH(sqlc.arg(entity_id)) = 0 OR entity_id = sqlc.arg(entity_id)) - AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) - AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) - AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) - -- TODO(mvdan): consider keeping an id_hex column for faster searches - AND (sqlc.arg(id_substr) = '' OR (INSTR(LOWER(HEX(id)), sqlc.arg(id_substr)) > 0)) - AND (sqlc.arg(with_results) = FALSE OR have_results) +WITH results AS ( + SELECT *, + COUNT(*) OVER() AS total_count + FROM processes + WHERE (LENGTH(sqlc.arg(entity_id)) = 0 OR entity_id = sqlc.arg(entity_id)) + AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) + AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) + AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) + -- TODO: consider keeping an id_hex column for faster searches + AND (sqlc.arg(id_substr) = '' OR (INSTR(LOWER(HEX(id)), sqlc.arg(id_substr)) > 0)) + AND (sqlc.arg(with_results) = FALSE OR have_results) +) +SELECT id, total_count +FROM results ORDER BY creation_time DESC, id ASC LIMIT sqlc.arg(limit) -OFFSET sqlc.arg(offset) -; +OFFSET sqlc.arg(offset); -- name: UpdateProcessFromState :execresult UPDATE processes @@ -96,13 +101,18 @@ SELECT COUNT(*) FROM processes; SELECT COUNT(DISTINCT entity_id) FROM processes; -- name: SearchEntities :many -SELECT entity_id, COUNT(id) AS process_count FROM processes -WHERE (sqlc.arg(entity_id_substr) = '' OR (INSTR(LOWER(HEX(entity_id)), sqlc.arg(entity_id_substr)) > 0)) +WITH results AS ( + SELECT *, + COUNT(*) OVER() AS total_count + FROM processes + WHERE (sqlc.arg(entity_id_substr) = '' OR (INSTR(LOWER(HEX(entity_id)), sqlc.arg(entity_id_substr)) > 0)) +) +SELECT entity_id, COUNT(id) AS process_count, total_count +FROM results GROUP BY entity_id ORDER BY creation_time DESC, id ASC LIMIT sqlc.arg(limit) -OFFSET sqlc.arg(offset) -; +OFFSET sqlc.arg(offset); -- name: GetProcessIDsByFinalResults :many SELECT id FROM processes