-
Notifications
You must be signed in to change notification settings - Fork 105
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
test: simulation import and export #3033
base: develop
Are you sure you want to change the base?
Changes from all commits
c3299ef
7d95021
fcce58e
b774b1e
ac8fddf
b55f439
913aa19
10d8a38
49d430c
73fbcf5
cad3f4f
c07d519
4f3ebbe
56b4698
c1db4cf
d98add5
bff4121
74b0ad7
f8c095d
b49be6b
cf4cb45
d5ae57c
1f744f8
e1cc91f
d674455
651a61a
5e3bce7
3b44c10
6eea359
9dfc005
a40f4d9
fdbacce
a868450
924d063
bea00ab
21b91d4
59bc886
33bb1b9
9791e99
643490d
064e05e
57b5d4e
6fcad07
4b0dda4
e8d127a
d18978b
195bc8d
7458f94
399569d
cf60f5a
5c682e9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,91 @@ | ||||||
name: sim | ||||||
|
||||||
on: | ||||||
push: | ||||||
branches: | ||||||
- develop | ||||||
pull_request: | ||||||
schedule: | ||||||
- cron: "0 6 * * *" | ||||||
workflow_dispatch: | ||||||
inputs: | ||||||
make-targets: | ||||||
description: 'Comma separated list of make targets to run (e.g., test-sim-nondeterminism, test-sim-fullappsimulation)' | ||||||
required: true | ||||||
default: '' | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider placing a default value here |
||||||
|
||||||
concurrency: | ||||||
group: simulation-${{ github.head_ref || github.sha }} | ||||||
cancel-in-progress: true | ||||||
|
||||||
jobs: | ||||||
matrix-conditionals: | ||||||
runs-on: ubuntu-22.04 | ||||||
outputs: | ||||||
SIM_TEST_NOND: ${{ steps.matrix-conditionals.outputs.SIM_TEST_NOND }} | ||||||
SIM_TEST_FULL: ${{ steps.matrix-conditionals.outputs.SIM_TEST_FULL }} | ||||||
SIM_TEST_IMPORT_EXPORT: ${{ steps.matrix-conditionals.outputs.SIM_TEST_IMPORT_EXPORT }} | ||||||
SIM_TEST_AFTER_IMPORT: ${{ steps.matrix-conditionals.outputs.SIM_TEST_AFTER_IMPORT }} | ||||||
steps: | ||||||
- id: matrix-conditionals | ||||||
uses: actions/github-script@v7 | ||||||
with: | ||||||
script: | | ||||||
const makeTargetsInput = context.payload.inputs ? context.payload.inputs['make-targets'] : null; | ||||||
const defaultTargets = ['test-sim-nondeterminism', 'test-sim-fullappsimulation', 'test-sim-import-export', 'test-sim-after-import']; | ||||||
|
||||||
const makeTargets = makeTargetsInput ? makeTargetsInput.split(',') : defaultTargets; | ||||||
|
||||||
core.setOutput('SIM_TEST_NOND', makeTargets.includes('test-sim-nondeterminism')); | ||||||
core.setOutput('SIM_TEST_FULL', makeTargets.includes('test-sim-fullappsimulation')); | ||||||
core.setOutput('SIM_TEST_IMPORT_EXPORT', makeTargets.includes('test-sim-import-export')); | ||||||
core.setOutput('SIM_TEST_AFTER_IMPORT', makeTargets.includes('test-sim-after-import')); | ||||||
|
||||||
simulation-tests: | ||||||
needs: matrix-conditionals | ||||||
runs-on: ubuntu-22.04 | ||||||
strategy: | ||||||
fail-fast: false | ||||||
matrix: | ||||||
include: | ||||||
- make-target: "test-sim-nondeterminism" | ||||||
condition: ${{ needs.matrix-conditionals.outputs.SIM_TEST_NOND == 'true' }} | ||||||
- make-target: "test-sim-fullappsimulation" | ||||||
condition: ${{ needs.matrix-conditionals.outputs.SIM_TEST_FULL == 'true' }} | ||||||
- make-target: "test-sim-import-export" | ||||||
condition: ${{ needs.matrix-conditionals.outputs.SIM_TEST_IMPORT_EXPORT == 'true' }} | ||||||
- make-target: "test-sim-after-import" | ||||||
condition: ${{ needs.matrix-conditionals.outputs.SIM_TEST_AFTER_IMPORT == 'true' }} | ||||||
name: ${{ matrix.make-target }} | ||||||
steps: | ||||||
- name: Checkout code | ||||||
uses: actions/checkout@v3 | ||||||
|
||||||
- name: Set up Go | ||||||
uses: actions/setup-go@v4 | ||||||
with: | ||||||
go-version: '1.21' | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
||||||
- name: Install dependencies | ||||||
run: make runsim | ||||||
|
||||||
- name: Run ${{ matrix.make-target }} | ||||||
if: ${{ matrix.condition }} | ||||||
run: | | ||||||
make ${{ matrix.make-target }} | ||||||
|
||||||
sim-ok: | ||||||
runs-on: ubuntu-22.04 | ||||||
needs: | ||||||
- matrix-conditionals | ||||||
- simulation-tests | ||||||
if: always() | ||||||
steps: | ||||||
- name: Aggregate Results | ||||||
run: | | ||||||
result="${{ needs.simulation-tests.result }}" | ||||||
if [[ $result == "success" || $result == "skipped" ]]; then | ||||||
exit 0 | ||||||
else | ||||||
exit 1 | ||||||
fi |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -75,10 +75,7 @@ func (app *App) prepForZeroHeightGenesis(ctx sdk.Context, jailAllowedAddrs []str | |
|
||
// withdraw all validator commission | ||
app.StakingKeeper.IterateValidators(ctx, func(_ int64, val stakingtypes.ValidatorI) (stop bool) { | ||
_, err := app.DistrKeeper.WithdrawValidatorCommission(ctx, val.GetOperator()) | ||
if err != nil { | ||
panic(err) | ||
} | ||
_, _ = app.DistrKeeper.WithdrawValidatorCommission(ctx, val.GetOperator()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can ignore the error as some validators might not have any commsion
kingpinXD marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return false | ||
}) | ||
|
||
|
@@ -162,7 +159,12 @@ func (app *App) prepForZeroHeightGenesis(ctx sdk.Context, jailAllowedAddrs []str | |
counter := int16(0) | ||
|
||
for ; iter.Valid(); iter.Next() { | ||
addr := sdk.ValAddress(iter.Key()[1:]) | ||
key := iter.Key() | ||
if len(key) <= 2 { | ||
app.Logger().Error("unexpected key in staking store", "key", key) | ||
continue | ||
} | ||
addr := sdk.ValAddress(key[2:]) | ||
validator, found := app.StakingKeeper.GetValidator(ctx, addr) | ||
if !found { | ||
panic("expected validator, not found") | ||
|
kingpinXD marked this conversation as resolved.
Show resolved
Hide resolved
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,34 +1,69 @@ | ||
# Zetachain simulation testing | ||
## Overview | ||
The blockchain simulation tests how the blockchain application would behave under real life circumstances by generating | ||
and sending randomized messages.The goal of this is to detect and debug failures that could halt a live chain,by providing | ||
logs and statistics about the operations run by the simulator as well as exporting the latest application state. | ||
|
||
and sending randomized messages.The goal of this is to detect and debug failures that could halt a live chain by | ||
providing logs and statistics about the operations run by the simulator as well as exporting the latest application | ||
state. | ||
|
||
## Simulation tests | ||
|
||
### Nondeterminism test | ||
Nondeterminism test runs a full application simulation , and produces multiple blocks as per the config | ||
It checks the determinism of the application by comparing the apphash at the end of each run to other runs | ||
The test certifies that , for the same set of operations ( irrespective of what the operations are ), we would reach the same final state if the initial state is the same | ||
The test certifies that, for the same set of operations (regardless of what the operations are), we | ||
would reach the same final state if the initial state is the same | ||
```bash | ||
make test-sim-nondeterminism | ||
``` | ||
|
||
### Full application simulation test | ||
Full application runs a full app simulation test with the provided configuration. | ||
At the end of the run it tries to export the genesis state to make sure the export works. | ||
At the end of the run, it tries to export the genesis state to make sure the export works. | ||
```bash | ||
make test-sim-full-app | ||
``` | ||
|
||
### Import Export simulation test | ||
The import export simulation test runs a full application simulation | ||
and exports the application state at the end of the run. | ||
This state is then imported into a new simulation. | ||
At the end of the run, we compare the keys for the application state for both the simulations | ||
to make sure they are the same. | ||
```bash | ||
make test-sim-import-export | ||
``` | ||
|
||
### Import and run simulation test | ||
This simulation test exports the application state at the end of the run and imports it into a new simulation. | ||
```bash | ||
make test-sim-after-import | ||
``` | ||
|
||
### Multi seed long test | ||
Multi seed long test runs a full application simulation with multiple seeds and multiple blocks.This runs the test for a longer duration compared to the multi seed short test | ||
Multi seed long test runs a full application simulation with multiple seeds and multiple blocks. | ||
This runs the test for a longer duration compared to the multi seed short test. | ||
```bash | ||
make test-sim-multi-seed-long | ||
``` | ||
|
||
### Multi seed short test | ||
Multi seed short test runs a full application simulation with multiple seeds and multiple blocks. This runs the test for a longer duration compared to the multi seed long test | ||
Multi seed short test runs a full application simulation with multiple seeds and multiple blocks. | ||
This runs the test for a shorter duration compared to the multi seed long test. | ||
```bash | ||
make test-sim-multi-seed-short | ||
``` | ||
``` | ||
|
||
kingpinXD marked this conversation as resolved.
Show resolved
Hide resolved
|
||
### Import Export long test | ||
This test runs the import export simulation test for a longer duration. | ||
It uses the `runsim` tool to run the same test in parallel threads. | ||
```bash | ||
make test-sim-import-export-long | ||
``` | ||
|
||
### Import and run simulation test long | ||
This test runs the import and run simulation test for a longer duration. | ||
It uses the `runsim` tool to run the same test in parallel threads. | ||
```bash | ||
make test-sim-after-import-long | ||
``` | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,12 +1,16 @@ | ||
package sim | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we need a directory I think this can also be removed from test coverage There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was planning to have a directory test that can hold all test-related files , and have directories for simulation, e2e etc There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would do
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would keep |
||
|
||
import ( | ||
"encoding/json" | ||
"fmt" | ||
"os" | ||
|
||
dbm "github.com/cometbft/cometbft-db" | ||
"github.com/cometbft/cometbft/libs/log" | ||
"github.com/cosmos/cosmos-sdk/baseapp" | ||
"github.com/cosmos/cosmos-sdk/runtime" | ||
servertypes "github.com/cosmos/cosmos-sdk/server/types" | ||
simtypes "github.com/cosmos/cosmos-sdk/types/simulation" | ||
"github.com/zeta-chain/ethermint/app" | ||
evmante "github.com/zeta-chain/ethermint/app/ante" | ||
|
||
|
@@ -66,3 +70,30 @@ func PrintStats(db dbm.DB) { | |
fmt.Println(db.Stats()["leveldb.stats"]) | ||
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) | ||
} | ||
|
||
// CheckExportSimulation exports the app state and simulation parameters to JSON | ||
// if the export paths are defined. | ||
func CheckExportSimulation(app runtime.AppI, config simtypes.Config, params simtypes.Params) error { | ||
if config.ExportStatePath != "" { | ||
exported, err := app.ExportAppStateAndValidators(false, nil, nil) | ||
if err != nil { | ||
return fmt.Errorf("failed to export app state: %w", err) | ||
} | ||
|
||
if err := os.WriteFile(config.ExportStatePath, exported.AppState, 0o600); err != nil { | ||
return err | ||
} | ||
} | ||
|
||
if config.ExportParamsPath != "" { | ||
paramsBz, err := json.MarshalIndent(params, "", " ") | ||
if err != nil { | ||
return fmt.Errorf("failed to write app state to %s: %w", config.ExportStatePath, err) | ||
} | ||
|
||
if err := os.WriteFile(config.ExportParamsPath, paramsBz, 0o600); err != nil { | ||
return err | ||
} | ||
} | ||
return nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I doubt running these tests every PR. Think of
zetaclient
PRs - most of the time they don't touchzetacore
part. Probably we should do the following:x/...
files were touchedSIM_TESTS
labelOtherwise the CI will become even slower