From 0d8ea5781295ab56bf35f77f2a462b9e866a7a7c Mon Sep 17 00:00:00 2001 From: Gary Malouf <982483+gmalouf@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:38:38 -0500 Subject: [PATCH] Support generating synthetic payout transaction based off Payout being set in block header. --- e2e_tests/docker/indexer/Dockerfile | 2 +- idb/postgres/internal/writer/write_txn.go | 52 +++- .../writer/write_txn_participation.go | 9 + idb/postgres/internal/writer/writer_test.go | 252 +++++++++++++++++- idb/postgres/postgres.go | 3 +- 5 files changed, 311 insertions(+), 7 deletions(-) diff --git a/e2e_tests/docker/indexer/Dockerfile b/e2e_tests/docker/indexer/Dockerfile index 754e1e8b4..50bff4ae1 100644 --- a/e2e_tests/docker/indexer/Dockerfile +++ b/e2e_tests/docker/indexer/Dockerfile @@ -1,4 +1,4 @@ -ARG GO_IMAGE=golang:1.213.3 +ARG GO_IMAGE=golang:1.23.3 FROM $GO_IMAGE ARG CHANNEL=stable diff --git a/idb/postgres/internal/writer/write_txn.go b/idb/postgres/internal/writer/write_txn.go index 4a6be5f0b..4ee769839 100644 --- a/idb/postgres/internal/writer/write_txn.go +++ b/idb/postgres/internal/writer/write_txn.go @@ -28,7 +28,7 @@ func transactionAssetID(stxnad *types.SignedTxnWithAD, intra uint, block *types. case types.ApplicationCallTx: assetid = uint64(stxnad.Txn.ApplicationID) if assetid == 0 { - assetid = uint64(stxnad.ApplyData.ApplicationID) + assetid = stxnad.ApplyData.ApplicationID } if assetid == 0 { if block == nil { @@ -42,7 +42,7 @@ func transactionAssetID(stxnad *types.SignedTxnWithAD, intra uint, block *types. case types.AssetConfigTx: assetid = uint64(stxnad.Txn.ConfigAsset) if assetid == 0 { - assetid = uint64(stxnad.ApplyData.ConfigAsset) + assetid = stxnad.ApplyData.ConfigAsset } if assetid == 0 { if block == nil { @@ -112,6 +112,7 @@ func yieldInnerTransactions(ctx context.Context, stxnad *types.SignedTxnWithAD, // Writes database rows for transactions (including inner transactions) to `outCh`. func yieldTransactions(ctx context.Context, block *types.Block, modifiedTxns []types.SignedTxnInBlock, outCh chan []interface{}) error { intra := uint(0) + for idx, stib := range block.Payset { var stxnad types.SignedTxnWithAD var err error @@ -123,7 +124,7 @@ func yieldTransactions(ctx context.Context, block *types.Block, modifiedTxns []t } txn := &stxnad.Txn - typeenum, ok := idb.GetTypeEnum(types.TxType(txn.Type)) + typeenum, ok := idb.GetTypeEnum(txn.Type) if !ok { return fmt.Errorf("yieldTransactions() get type enum") } @@ -153,9 +154,54 @@ func yieldTransactions(ctx context.Context, block *types.Block, modifiedTxns []t } } + if block.ProposerPayout > 0 { + stxnad := SignedTransactionFromBlockPayout(block) + typeenum, ok := idb.GetTypeEnum(stxnad.Txn.Type) + if !ok { + return fmt.Errorf("yieldTransactions() ProposerPayout get type enum - should NEVER happen") + } + id := crypto.TransactionIDString(stxnad.Txn) + extra := idb.TxnExtra{} + row := []interface{}{ + uint64(block.Round), intra, int(typeenum), 0, id, + encoding.EncodeSignedTxnWithAD(stxnad), + encoding.EncodeTxnExtra(&extra)} + select { + case <-ctx.Done(): + return fmt.Errorf("yieldTransactions() ProposerPayout ctx.Err(): %w", ctx.Err()) + case outCh <- row: + } + intra++ + } + return nil } +// SignedTransactionFromBlockPayout creates a synthetic transaction for the proposer payout. +func SignedTransactionFromBlockPayout(block *types.Block) types.SignedTxnWithAD { + stxnad := types.SignedTxnWithAD{ + SignedTxn: types.SignedTxn{ + Txn: types.Transaction{ + Type: types.PaymentTx, + Header: types.Header{ + Sender: block.FeeSink, + Note: []byte("ProposerPayout for Round " + fmt.Sprint(block.Round)), + FirstValid: block.Round, + LastValid: block.Round, + GenesisID: block.GenesisID, + GenesisHash: block.GenesisHash, + }, + PaymentTxnFields: types.PaymentTxnFields{ + Receiver: block.Proposer, + Amount: block.ProposerPayout, + }, + }, + }, + } + + return stxnad +} + // AddTransactions adds transactions from `block` to the database. // `modifiedTxns` contains enhanced apply data generated by evaluator. func AddTransactions(block *types.Block, modifiedTxns []types.SignedTxnInBlock, tx pgx.Tx) error { diff --git a/idb/postgres/internal/writer/write_txn_participation.go b/idb/postgres/internal/writer/write_txn_participation.go index 65edf1c1c..38b5add50 100644 --- a/idb/postgres/internal/writer/write_txn_participation.go +++ b/idb/postgres/internal/writer/write_txn_participation.go @@ -125,6 +125,15 @@ func AddTransactionParticipation(block *types.Block, tx pgx.Tx) error { next, rows = addInnerTransactionParticipation(&stxnib.SignedTxnWithAD, uint64(block.Round), next+1, rows) } + if block.ProposerPayout > 0 { + // FeeSink is the sender, Proposer is the receiver. + participants := []types.Address{block.FeeSink, block.Proposer} + for j := range participants { + rows = append(rows, []interface{}{participants[j][:], uint64(block.Round), next}) + } + next++ + } + _, err := tx.CopyFrom( context.Background(), pgx.Identifier{"txn_participation"}, diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index d793b97a4..e0edcbfb8 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -168,7 +168,7 @@ func TestWriterSpecialAccounts(t *testing.T) { assert.Equal(t, expected, accounts) } -func TestWriterTxnTableBasic(t *testing.T) { +func TestWriterTxnTableBasicNoPayout(t *testing.T) { db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() @@ -255,6 +255,116 @@ func TestWriterTxnTableBasic(t *testing.T) { assert.NoError(t, rows.Err()) } +func TestWriterTxnTableBasicWithProposalPayout(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + block := sdk.Block{ + BlockHeader: sdk.BlockHeader{ + Round: sdk.Round(2), + TimeStamp: 333, + GenesisID: test.MakeGenesis().ID(), + GenesisHash: test.MakeGenesis().Hash(), + RewardsState: sdk.RewardsState{ + FeeSink: test.FeeAddr, + RewardsLevel: 111111, + }, + TxnCounter: 9, + UpgradeState: sdk.UpgradeState{ + CurrentProtocol: "future", + }, + Proposer: test.AccountE, + ProposerPayout: 2000000, + }, + Payset: make([]sdk.SignedTxnInBlock, 2), + } + + stxnad0 := test.MakePaymentTxn( + 1000, 1, 0, 0, 0, 0, sdk.Address(test.AccountA), sdk.Address(test.AccountB), sdk.Address{}, + sdk.Address{}) + var err error + block.Payset[0], err = + util.EncodeSignedTxn(block.BlockHeader, stxnad0.SignedTxn, stxnad0.ApplyData) + require.NoError(t, err) + + stxnad1 := test.MakeAssetConfigTxn( + 0, 100, 1, false, "ma", "myasset", "myasset.com", sdk.Address(test.AccountA)) + block.Payset[1], err = + util.EncodeSignedTxn(block.BlockHeader, stxnad1.SignedTxn, stxnad1.ApplyData) + require.NoError(t, err) + + f := func(tx pgx.Tx) error { + return writer.AddTransactions(&block, block.Payset, tx) + } + err = pgutil.TxWithRetry(db, serializable, f, nil) + require.NoError(t, err) + + rows, err := db.Query(context.Background(), "SELECT * FROM txn ORDER BY intra") + require.NoError(t, err) + defer rows.Close() + + var round uint64 + var intra uint64 + var typeenum uint + var asset uint64 + var txid []byte + var txn []byte + var extra []byte + + require.True(t, rows.Next()) + err = rows.Scan(&round, &intra, &typeenum, &asset, &txid, &txn, &extra) + require.NoError(t, err) + assert.Equal(t, block.Round, sdk.Round(round)) + assert.Equal(t, uint64(0), intra) + assert.Equal(t, idb.TypeEnumPay, idb.TxnTypeEnum(typeenum)) + assert.Equal(t, uint64(0), asset) + assert.Equal(t, crypto2.TransactionIDString(stxnad0.Txn), string(txid)) + { + stxn, err := encoding.DecodeSignedTxnWithAD(txn) + require.NoError(t, err) + assert.Equal(t, stxnad0, stxn) + } + assert.Equal(t, "{}", string(extra)) + + require.True(t, rows.Next()) + err = rows.Scan(&round, &intra, &typeenum, &asset, &txid, &txn, &extra) + require.NoError(t, err) + assert.Equal(t, block.Round, sdk.Round(round)) + assert.Equal(t, uint64(1), intra) + assert.Equal(t, idb.TypeEnumAssetConfig, idb.TxnTypeEnum(typeenum)) + assert.Equal(t, uint64(9), asset) + assert.Equal(t, crypto2.TransactionIDString(stxnad1.Txn), string(txid)) + { + stxn, err := encoding.DecodeSignedTxnWithAD(txn) + require.NoError(t, err) + assert.Equal(t, stxnad1, stxn) + } + assert.Equal(t, "{}", string(extra)) + + // Payout should be the last transaction. + require.True(t, rows.Next()) + err = rows.Scan(&round, &intra, &typeenum, &asset, &txid, &txn, &extra) + require.NoError(t, err) + + assert.Equal(t, block.Round, sdk.Round(round)) + assert.Equal(t, uint64(2), intra) + assert.Equal(t, idb.TypeEnumPay, idb.TxnTypeEnum(typeenum)) + assert.Equal(t, uint64(0), asset) + + // Intentionally using synthetic payout transaction logic; we are testing insertion and validity + payoutTxn := writer.SignedTransactionFromBlockPayout(&block) + assert.Equal(t, crypto2.TransactionIDString(payoutTxn.Txn), string(txid)) + { + stxn, err := encoding.DecodeSignedTxnWithAD(txn) + require.NoError(t, err) + assert.Equal(t, payoutTxn, stxn) + } + assert.Equal(t, "{}", string(extra)) + + assert.False(t, rows.Next()) + assert.NoError(t, rows.Err()) +} + // Test that asset close amount is written even if it is missing in the apply data // in the block (it is present in the "modified transactions"). func TestWriterTxnTableAssetCloseAmount(t *testing.T) { @@ -314,7 +424,7 @@ func TestWriterTxnTableAssetCloseAmount(t *testing.T) { assert.NoError(t, rows.Err()) } -func TestWriterTxnParticipationTable(t *testing.T) { +func TestWriterTxnParticipationTableNoPayout(t *testing.T) { type testtype struct { name string payset sdk.Payset @@ -425,6 +535,144 @@ func TestWriterTxnParticipationTable(t *testing.T) { } } +func TestWriterTxnParticipationTableWithPayout(t *testing.T) { + type testtype struct { + name string + payset sdk.Payset + expected []txnParticipationRow + } + + makeBlockFunc := func() sdk.Block { + return sdk.Block{ + BlockHeader: sdk.BlockHeader{ + Round: sdk.Round(2), + GenesisID: test.MakeGenesis().ID(), + GenesisHash: test.MakeGenesis().Hash(), + RewardsState: sdk.RewardsState{ + FeeSink: test.FeeAddr, + }, + UpgradeState: sdk.UpgradeState{ + CurrentProtocol: "future", + }, + Proposer: test.AccountE, + ProposerPayout: 2000000, + }, + } + } + + var tests []testtype + { + stxnad0 := test.MakePaymentTxn( + 1000, 1, 0, 0, 0, 0, sdk.Address(test.AccountA), sdk.Address(test.AccountB), sdk.Address{}, + sdk.Address{}) + stib0, err := util.EncodeSignedTxn(makeBlockFunc().BlockHeader, stxnad0.SignedTxn, stxnad0.ApplyData) + require.NoError(t, err) + + stxnad1 := test.MakeAssetConfigTxn( + 0, 100, 1, false, "ma", "myasset", "myasset.com", sdk.Address(test.AccountC)) + stib1, err := util.EncodeSignedTxn(makeBlockFunc().BlockHeader, stxnad1.SignedTxn, stxnad1.ApplyData) + require.NoError(t, err) + + testcase := testtype{ + name: "basic", + payset: []sdk.SignedTxnInBlock{stib0, stib1}, + expected: []txnParticipationRow{ + { + addr: test.AccountA, + round: 2, + intra: 0, + }, + { + addr: test.AccountB, + round: 2, + intra: 0, + }, + { + addr: test.AccountC, + round: 2, + intra: 1, + }, + // Payout involved accounts + { + addr: test.AccountE, + round: 2, + intra: 2, + }, + { + addr: test.FeeAddr, + round: 2, + intra: 2, + }, + }, + } + tests = append(tests, testcase) + } + { + stxnad := test.MakeCreateAppTxn(sdk.Address(test.AccountA)) + stxnad.Txn.ApplicationCallTxnFields.Accounts = + []sdk.Address{sdk.Address(test.AccountB), sdk.Address(test.AccountC)} + stib, err := util.EncodeSignedTxn(makeBlockFunc().BlockHeader, stxnad.SignedTxn, stxnad.ApplyData) + require.NoError(t, err) + + testcase := testtype{ + name: "app_call_addresses", + payset: []sdk.SignedTxnInBlock{stib}, + expected: []txnParticipationRow{ + { + addr: sdk.Address(test.AccountA), + round: 2, + intra: 0, + }, + { + addr: sdk.Address(test.AccountB), + round: 2, + intra: 0, + }, + { + addr: sdk.Address(test.AccountC), + round: 2, + intra: 0, + }, + // Payout involved accounts + { + addr: test.AccountE, + round: 2, + intra: 1, + }, + { + addr: test.FeeAddr, + round: 2, + intra: 1, + }, + }, + } + tests = append(tests, testcase) + } + + for _, testcase := range tests { + t.Run(testcase.name, func(t *testing.T) { + db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) + defer shutdownFunc() + + block := makeBlockFunc() + block.Payset = testcase.payset + + f := func(tx pgx.Tx) error { + return writer.AddTransactionParticipation(&block, tx) + } + err := pgutil.TxWithRetry(db, serializable, f, nil) + require.NoError(t, err) + + results, err := txnParticipationQuery( + db, `SELECT * FROM txn_participation ORDER BY round, intra, addr`) + assert.NoError(t, err) + + // Verify expected participation + assert.Equal(t, testcase.expected, results) + }) + } +} + // Create a new account and then delete it. func TestWriterAccountTableBasic(t *testing.T) { db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 1a6850ccb..777c4f9c5 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -698,8 +698,9 @@ func buildTransactionQuery(tf idb.TransactionFilter) (query string, whereArgs [] whereStr := strings.Join(whereParts, " AND ") query += " WHERE " + whereStr } + if joinParticipation { - // this should match the index on txn_particpation + // this should match the index on txn_participation query += " ORDER BY p.addr, p.round DESC, p.intra DESC" } else { // this should explicitly match the primary key on txn (round,intra)