Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: move rate limiter to zetaclient and add metrics #2110

Merged
merged 50 commits into from
May 7, 2024
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
8db24cc
initial commit of grpc pending cctx query with rate limiter
ws4charlie Apr 18, 2024
a873173
replace big.Float with sdk.Dec and update mock rate limiter flags
ws4charlie Apr 18, 2024
60cdbd0
split big loop into backwards loop and forwards loop to be more accurate
ws4charlie Apr 18, 2024
b888133
Merge branch 'develop' of https://github.com/zeta-chain/node into fea…
ws4charlie Apr 18, 2024
3f63f22
adjust zetaclient code to query pending cctx with rate limit
ws4charlie Apr 18, 2024
f0b0677
update change log and add one more rate limiter flag test
ws4charlie Apr 19, 2024
aab441e
use outboun amount for calculation
ws4charlie Apr 19, 2024
00ab5ac
some minimum code refactor
ws4charlie Apr 19, 2024
f31c988
created separate file for cctx query with rate limit
ws4charlie Apr 19, 2024
3cdf614
Merge branch 'develop' into feat-withdraw-rate-limit
ws4charlie Apr 19, 2024
6ad94af
improved a few error handlling
ws4charlie Apr 19, 2024
27e94a7
use old cctx query as fallback when rate limiter is disabled; some re…
ws4charlie Apr 19, 2024
0eb3b8a
fixed unit test compile
ws4charlie Apr 19, 2024
1568e44
Merge branch 'develop' into feat-withdraw-rate-limit
lumtis Apr 19, 2024
60e5dcc
added unit test for fallback query
ws4charlie Apr 20, 2024
101457c
added unit tests for cctx value conversion
ws4charlie Apr 20, 2024
7065d2e
added unit tests for cctx value conversion
ws4charlie Apr 20, 2024
874d5f3
add changelog entry
ws4charlie Apr 20, 2024
16955b8
added unit tests for query pending cctxs within rate limit
ws4charlie Apr 20, 2024
249bcaa
added total value in rate limiter window for monitoring purpose
ws4charlie Apr 20, 2024
dd950a8
Merge branch 'develop' into unit-test-rate-limit
lumtis Apr 22, 2024
7fe367f
Update x/crosschain/keeper/grpc_query_cctx_rate_limit.go
ws4charlie Apr 22, 2024
e5e4e28
change variable name fCoin to foreignCoin
ws4charlie Apr 22, 2024
5e28daf
Update x/fungible/keeper/foreign_coins.go
ws4charlie Apr 22, 2024
da27a84
Update x/crosschain/keeper/grpc_query_cctx_rate_limit_test.go
ws4charlie Apr 22, 2024
f9eed15
converted rate limiter query unit tests to table test
ws4charlie Apr 22, 2024
f40cf45
handle edge case when pending cctxs span wider block range than slidi…
ws4charlie Apr 24, 2024
578afcc
added zero rate check; added comment to make unit test clearer
ws4charlie Apr 24, 2024
fa24f32
added unit test and note for method GetAllForeignCoinMap
ws4charlie Apr 24, 2024
61627b1
treat Rate as average block rate; stop outbound when current rate lim…
ws4charlie Apr 25, 2024
514cea7
refactor: allow zeta deposits to new zevm address (#2076)
kingpinXD Apr 25, 2024
daba269
add commented unit tests back
ws4charlie Apr 25, 2024
ec6aca8
Merge branch 'v16.0.0-rc' into unit-test-rate-limit
lumtis Apr 29, 2024
c011900
replace sdk.Dec with sdkmath.Int to represent cctx value in azeta
ws4charlie Apr 29, 2024
bd13759
test(e2e): add rate limiter admin E2E test (#2063)
lumtis Apr 29, 2024
4be83fb
removed incorrect Note
ws4charlie Apr 29, 2024
a450d91
initiated rate limiter refactor and added metrics
ws4charlie May 2, 2024
c3c885b
added more unit tests and updated changelog
ws4charlie May 2, 2024
21a2076
print more details from rate limiter output
ws4charlie May 2, 2024
e8c0c0a
Merge branch 'develop' of https://github.com/zeta-chain/node into ref…
ws4charlie May 2, 2024
cd3e003
reorder observer methods declaration
ws4charlie May 2, 2024
e5c4c1f
fix unit test
ws4charlie May 3, 2024
77acc5b
moved state irrelevant methods to types; renaming and cleaning
ws4charlie May 3, 2024
fab00cd
code indentation and update variable
ws4charlie May 3, 2024
b4904b4
Merge branch 'develop' of https://github.com/zeta-chain/node into ref…
ws4charlie May 3, 2024
56d2cc1
replace with
ws4charlie May 3, 2024
6b04e01
fix unit test
ws4charlie May 3, 2024
f63f264
added extra unit tests and improved func name
ws4charlie May 7, 2024
e7ae411
Merge branch 'develop' into refactor-rate-limiter
ws4charlie May 7, 2024
ebc87ee
Merge branch 'develop' into refactor-rate-limiter
ws4charlie May 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
* [2013](https://github.com/zeta-chain/node/pull/2013) - rename `GasPriceVoter` message to `VoteGasPrice`
* [2059](https://github.com/zeta-chain/node/pull/2059) - Remove unused params from all functions in zetanode
* [2076](https://github.com/zeta-chain/node/pull/2076) - automatically deposit native zeta to an address if it doesn't exist on ZEVM.
* [2110](https://github.com/zeta-chain/node/pull/2110) - move non-query rate limiter logic to zetaclient side and code refactor.
ws4charlie marked this conversation as resolved.
Show resolved Hide resolved
* [2071](https://github.com/zeta-chain/node/pull/2071) - Modify chains struct to add all chain related information

### Features
Expand Down
1 change: 1 addition & 0 deletions cmd/zetae2e/local/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
e2etests.TestUpdateBytecodeZRC20Name,
e2etests.TestUpdateBytecodeConnectorName,
e2etests.TestDepositEtherLiquidityCapName,
e2etests.TestRateLimiterName,
ws4charlie marked this conversation as resolved.
Show resolved Hide resolved

// TestMigrateChainSupportName tests EVM chain migration. Currently this test doesn't work with Anvil because pre-EIP1559 txs are not supported
// See issue below for details
Expand Down
52 changes: 52 additions & 0 deletions docs/openapi/openapi.swagger.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27223,6 +27223,32 @@ paths:
$ref: '#/definitions/googlerpcStatus'
tags:
- Query
/zeta-chain/crosschain/rateLimiterInput:
get:
summary: Queries the input data of rate limiter.
operationId: Query_RateLimiterInput
responses:
"200":
description: A successful response.
schema:
$ref: '#/definitions/crosschainQueryRateLimiterInputResponse'
default:
description: An unexpected error response.
schema:
$ref: '#/definitions/googlerpcStatus'
parameters:
- name: limit
in: query
required: false
type: integer
format: int64
- name: window
in: query
required: false
type: string
format: int64
tags:
- Query
/zeta-chain/crosschain/zetaAccounting:
get:
operationId: Query_ZetaAccounting
Expand Down Expand Up @@ -54077,6 +54103,32 @@ definitions:
properties:
rateLimiterFlags:
$ref: '#/definitions/crosschainRateLimiterFlags'
crosschainQueryRateLimiterInputResponse:
type: object
properties:
height:
type: string
format: int64
cctxs_missed:
type: array
items:
type: object
$ref: '#/definitions/crosschainCrossChainTx'
cctxs_pending:
type: array
items:
type: object
$ref: '#/definitions/crosschainCrossChainTx'
total_pending:
type: string
format: uint64
past_cctxs_value:
type: string
pending_cctxs_value:
type: string
lowest_pending_cctx_height:
type: string
format: int64
crosschainQueryZetaAccountingResponse:
type: object
properties:
Expand Down
23 changes: 23 additions & 0 deletions pkg/math/float.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package math

import (
"math/big"
)

// Percentage calculates the percentage of A over B.
func Percentage(a, b *big.Int) *big.Float {
// if a is zero, return nil to avoid division by zero
if b.Cmp(big.NewInt(0)) == 0 {
return nil
}

// convert a and a to big.Float
ws4charlie marked this conversation as resolved.
Show resolved Hide resolved
floatA := new(big.Float).SetInt(a)
floatB := new(big.Float).SetInt(b)

// calculate the percentage of a over b
percentage := new(big.Float).Quo(floatA, floatB)
percentage.Mul(percentage, big.NewFloat(100))

return percentage
}
54 changes: 54 additions & 0 deletions pkg/math/float_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package math

import (
"fmt"
"math/big"
"testing"

"github.com/stretchr/testify/require"
)

func TestPercentage(t *testing.T) {
testCases := []struct {
name string
numerator *big.Int
denominator *big.Int
percentage *big.Float
fail bool
}{
{
name: "positive percentage",
numerator: big.NewInt(165),
denominator: big.NewInt(1000),
percentage: big.NewFloat(16.5),
fail: false,
},
{
name: "negative percentage",
numerator: big.NewInt(-165),
denominator: big.NewInt(1000),
percentage: big.NewFloat(-16.5),
fail: false,
},
{
name: "zero denominator",
numerator: big.NewInt(1),
denominator: big.NewInt(0),
percentage: nil,
fail: true,
},
}

for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
percentage := Percentage(tc.numerator, tc.denominator)
fmt.Printf("percentage: %v\n", percentage)
if tc.fail {
require.Nil(t, percentage)
} else {
require.True(t, percentage.Cmp(tc.percentage) == 0)
}
})
}
}
20 changes: 20 additions & 0 deletions proto/crosschain/query.proto
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ service Query {
rpc RateLimiterFlags(QueryRateLimiterFlagsRequest) returns (QueryRateLimiterFlagsResponse) {
option (google.api.http).get = "/zeta-chain/crosschain/rateLimiterFlags";
}

// Queries the input data of rate limiter.
rpc RateLimiterInput(QueryRateLimiterInputRequest) returns (QueryRateLimiterInputResponse) {
option (google.api.http).get = "/zeta-chain/crosschain/rateLimiterInput";
}
}

message QueryZetaAccountingRequest {}
Expand Down Expand Up @@ -263,6 +268,21 @@ message QueryListPendingCctxResponse {
uint64 totalPending = 2;
}

message QueryRateLimiterInputRequest {
uint32 limit = 1;
int64 window = 2;
}

message QueryRateLimiterInputResponse {
int64 height = 1;
repeated CrossChainTx cctxs_missed = 2;
repeated CrossChainTx cctxs_pending = 3;
uint64 total_pending = 4;
string past_cctxs_value = 5;
string pending_cctxs_value = 6;
int64 lowest_pending_cctx_height = 7;
}

message QueryListPendingCctxWithinRateLimitRequest {
uint32 limit = 1;
}
Expand Down
12 changes: 12 additions & 0 deletions proto/crosschain/rate_limiter_flags.proto
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ syntax = "proto3";
package zetachain.zetacore.crosschain;

import "gogoproto/gogo.proto";
import "pkg/coin/coin.proto";

option go_package = "github.com/zeta-chain/zetacore/x/crosschain/types";

Expand All @@ -28,3 +29,14 @@ message Conversion {
(gogoproto.nullable) = false
];
}

message AssetRate {
int64 chainId = 1;
string asset = 2;
uint32 decimals = 3;
coin.CoinType coin_type = 4;
string rate = 5 [
(gogoproto.customtype) = "github.com/cosmos/cosmos-sdk/types.Dec",
(gogoproto.nullable) = false
];
}
40 changes: 40 additions & 0 deletions testutil/sample/crosschain.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package sample
import (
"encoding/base64"
"encoding/json"
"fmt"
"math/rand"
"testing"

Expand Down Expand Up @@ -49,6 +50,18 @@ func RateLimiterFlags() types.RateLimiterFlags {
}
}

func AssetRate() *types.AssetRate {
r := Rand()

return &types.AssetRate{
ChainId: r.Int63(),
Asset: EthAddress().Hex(),
Decimals: uint32(r.Uint64()),
CoinType: coin.CoinType_ERC20,
Rate: sdk.NewDec(r.Int63()),
}
}

func OutTxTracker(t *testing.T, index string) types.OutTxTracker {
r := newRandFromStringSeed(t, index)

Expand Down Expand Up @@ -173,6 +186,33 @@ func CrossChainTx(t *testing.T, index string) *types.CrossChainTx {
}
}

// CustomCctxsInBlockRange create 1 cctx per block in block range [lowBlock, highBlock] (inclusive)
func CustomCctxsInBlockRange(
t *testing.T,
lowBlock uint64,
highBlock uint64,
chainID int64,
coinType coin.CoinType,
asset string,
amount uint64,
status types.CctxStatus,
) (cctxs []*types.CrossChainTx) {
// create 1 cctx per block
for i := lowBlock; i <= highBlock; i++ {
nonce := i - 1
cctx := CrossChainTx(t, fmt.Sprintf("%d-%d", chainID, nonce))
cctx.CctxStatus.Status = status
cctx.InboundTxParams.CoinType = coinType
cctx.InboundTxParams.Asset = asset
cctx.InboundTxParams.InboundTxObservedExternalHeight = i
cctx.GetCurrentOutTxParam().ReceiverChainId = chainID
cctx.GetCurrentOutTxParam().Amount = sdk.NewUint(amount)
cctx.GetCurrentOutTxParam().OutboundTxTssNonce = nonce
cctxs = append(cctxs, cctx)
}
return cctxs
}

func LastBlockHeight(t *testing.T, index string) *types.LastBlockHeight {
r := newRandFromStringSeed(t, index)

Expand Down
83 changes: 83 additions & 0 deletions typescript/crosschain/query_pb.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -871,6 +871,89 @@ export declare class QueryListPendingCctxResponse extends Message<QueryListPendi
static equals(a: QueryListPendingCctxResponse | PlainMessage<QueryListPendingCctxResponse> | undefined, b: QueryListPendingCctxResponse | PlainMessage<QueryListPendingCctxResponse> | undefined): boolean;
}

/**
* @generated from message zetachain.zetacore.crosschain.QueryRateLimiterInputRequest
*/
export declare class QueryRateLimiterInputRequest extends Message<QueryRateLimiterInputRequest> {
/**
* @generated from field: uint32 limit = 1;
*/
limit: number;

/**
* @generated from field: int64 window = 2;
*/
window: bigint;

constructor(data?: PartialMessage<QueryRateLimiterInputRequest>);

static readonly runtime: typeof proto3;
static readonly typeName = "zetachain.zetacore.crosschain.QueryRateLimiterInputRequest";
static readonly fields: FieldList;

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): QueryRateLimiterInputRequest;

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): QueryRateLimiterInputRequest;

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): QueryRateLimiterInputRequest;

static equals(a: QueryRateLimiterInputRequest | PlainMessage<QueryRateLimiterInputRequest> | undefined, b: QueryRateLimiterInputRequest | PlainMessage<QueryRateLimiterInputRequest> | undefined): boolean;
}

/**
* @generated from message zetachain.zetacore.crosschain.QueryRateLimiterInputResponse
*/
export declare class QueryRateLimiterInputResponse extends Message<QueryRateLimiterInputResponse> {
/**
* @generated from field: int64 height = 1;
*/
height: bigint;

/**
* @generated from field: repeated zetachain.zetacore.crosschain.CrossChainTx cctxs_missed = 2;
*/
cctxsMissed: CrossChainTx[];

/**
* @generated from field: repeated zetachain.zetacore.crosschain.CrossChainTx cctxs_pending = 3;
*/
cctxsPending: CrossChainTx[];

/**
* @generated from field: uint64 total_pending = 4;
*/
totalPending: bigint;

/**
* @generated from field: string past_cctxs_value = 5;
*/
pastCctxsValue: string;

/**
* @generated from field: string pending_cctxs_value = 6;
*/
pendingCctxsValue: string;

/**
* @generated from field: int64 lowest_pending_cctx_height = 7;
*/
lowestPendingCctxHeight: bigint;

constructor(data?: PartialMessage<QueryRateLimiterInputResponse>);

static readonly runtime: typeof proto3;
static readonly typeName = "zetachain.zetacore.crosschain.QueryRateLimiterInputResponse";
static readonly fields: FieldList;

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): QueryRateLimiterInputResponse;

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): QueryRateLimiterInputResponse;

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): QueryRateLimiterInputResponse;

static equals(a: QueryRateLimiterInputResponse | PlainMessage<QueryRateLimiterInputResponse> | undefined, b: QueryRateLimiterInputResponse | PlainMessage<QueryRateLimiterInputResponse> | undefined): boolean;
}

/**
* @generated from message zetachain.zetacore.crosschain.QueryListPendingCctxWithinRateLimitRequest
*/
Expand Down
Loading
Loading