Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests: separate e2e tests into a standalone module #1273

Closed
wants to merge 14 commits into from
Closed

tests: separate e2e tests into a standalone module #1273

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
c048452
chore: rm rosetta from cmd; rm direct require in go.mod
MSalopek Jul 25, 2023
487196b
experiment: update modfile; rm rosetta; rm e2e stuff
MSalopek Jul 25, 2023
ff0e7d9
experiment: enable e2e go.mod; currently broken
MSalopek Jul 25, 2023
6b8cc22
wip: rm go.work file due to deps issues
MSalopek Jul 26, 2023
1237330
rm old dockerfiles from e2e; update .gitignore
MSalopek Sep 7, 2023
940b440
refactor Makefile in root dir
MSalopek Sep 7, 2023
afe01a0
add instructions; rm go.work.sum
MSalopek Sep 7, 2023
4e11e59
force removal go.work.sum
MSalopek Sep 7, 2023
e70a9f1
fix broken link
MSalopek Sep 7, 2023
4fe67a2
commit go.work
MSalopek Sep 7, 2023
5b5e011
attempt to update deps on e2e and workfile
MSalopek Sep 8, 2023
bb3d7fd
rm ics/v3 as dep from e2e
MSalopek Sep 8, 2023
b85b3e1
Merge branch 'main' into masa/e2e-split-smaller-steps
MSalopek Sep 11, 2023
d947330
update Makefile gaia test runs
MSalopek Sep 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@ docs/tla/states/
vendor/
.vscode
.idea

# Go workspace files should not be pushed.
go.work
go.work.sum
4 changes: 2 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ COPY --from=is-builder /go/bin/interchain-security-cdd /usr/local/bin/interchain
COPY --from=is-builder /go/bin/interchain-security-sd /usr/local/bin/interchain-security-sd

# Copy in the shell scripts that run the testnet
ADD ./tests/e2e/testnet-scripts /testnet-scripts
ADD ./e2e/testnet-scripts /testnet-scripts

# Copy in the hermes config
ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml
ADD ./e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml
4 changes: 2 additions & 2 deletions Dockerfile.gaia
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ COPY --from=is-builder /go/bin/interchain-security-cdd /usr/local/bin/interchain
COPY --from=is-builder /go/bin/interchain-security-sd /usr/local/bin/interchain-security-sd

# Copy in the shell scripts that run the testnet
ADD ./tests/e2e/testnet-scripts /testnet-scripts
ADD ./e2e/testnet-scripts /testnet-scripts

# Copy in the hermes config
ADD ./tests/e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml
ADD ./e2e/testnet-scripts/hermes-config.toml /root/.hermes/config.toml
36 changes: 18 additions & 18 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ install: go.sum
export GOFLAGS='-buildmode=pie'
export CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2"
export CGO_LDFLAGS="-Wl,-z,relro,-z,now -fstack-protector"
go install $(BUILD_FLAGS) ./cmd/interchain-security-pd
go install $(BUILD_FLAGS) ./cmd/interchain-security-cd
go install $(BUILD_FLAGS) ./cmd/interchain-security-cdd
go install $(BUILD_FLAGS) ./cmd/interchain-security-sd
GOWORK='off' go install $(BUILD_FLAGS) ./cmd/interchain-security-pd
GOWORK='off' go install $(BUILD_FLAGS) ./cmd/interchain-security-cd
GOWORK='off' go install $(BUILD_FLAGS) ./cmd/interchain-security-cdd
GOWORK='off' go install $(BUILD_FLAGS) ./cmd/interchain-security-sd

# run all tests: unit, integration, diff, and E2E
test:
go test ./... && go run ./tests/e2e/...
test:
go test ./... && go run ./e2e/...

# run all unit tests
test-unit:
Expand All @@ -23,59 +23,59 @@ test-short:

# run E2E tests
test-e2e:
go run ./tests/e2e/...
go run ./e2e/...

# run difference tests
test-diff:
go test ./tests/difference/...

# run only happy path E2E tests
test-e2e-short:
go run ./tests/e2e/... --tc happy-path
go run ./e2e/... --tc happy-path

# run only happy path E2E tests with cometmock
# this set of traces does not test equivocation but it does check downtime
test-e2e-short-cometmock:
go run ./tests/e2e/... --tc happy-path-short --use-cometmock --use-gorelayer
go run ./e2e/... --tc happy-path-short --use-cometmock --use-gorelayer

# run full E2E tests in sequence (including multiconsumer)
test-e2e-multi-consumer:
go run ./tests/e2e/... --include-multi-consumer
go run ./e2e/... --include-multi-consumer

# run full E2E tests in parallel (including multiconsumer)
test-e2e-parallel:
go run ./tests/e2e/... --include-multi-consumer --parallel
go run ./e2e/... --include-multi-consumer --parallel

# run full E2E tests in sequence (including multiconsumer) using latest tagged gaia
test-gaia-e2e:
go run ./tests/e2e/... --include-multi-consumer --use-gaia
go run ./e2e/... --include-multi-consumer --use-gaia

# run only happy path E2E tests using latest tagged gaia
test-gaia-e2e-short:
go run ./tests/e2e/... --tc happy-path --use-gaia
go run ./e2e/... --happy-path-only --use-gaia
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
go run ./e2e/... --happy-path-only --use-gaia
go run ./e2e/... --tc happy-path --use-gaia


# run full E2E tests in parallel (including multiconsumer) using latest tagged gaia
test-gaia-e2e-parallel:
go run ./tests/e2e/... --include-multi-consumer --parallel --use-gaia
go run ./e2e/... --include-multi-consumer --parallel --use-gaia

# run full E2E tests in sequence (including multiconsumer) using specific tagged version of gaia
# usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-tagged
test-gaia-e2e-tagged:
go run ./tests/e2e/... --include-multi-consumer --use-gaia --gaia-tag $(GAIA_TAG)
go run ./e2e/... --include-multi-consumer --use-gaia --gaia-tag $(GAIA_TAG)

# run only happy path E2E tests using latest tagged gaia
# usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-short-tagged
test-gaia-e2e-short-tagged:
go run ./tests/e2e/... --tc happy-path --use-gaia --gaia-tag $(GAIA_TAG)
go run ./e2e/... --happy-path-only --use-gaia --gaia-tag $(GAIA_TAG)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here, what is the reason to go back to the 'old' flag and not using --tc
if I'm not mistaken that's not even supported anymore. did you test that?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That test has not been working for a while. Gaia is on v45, the ICS main is on v47.
Since e2e was coupled to the ICS version, some of the commands to execute against a gaia node are not valid and cannot be executed. Notice that most of that was intended for gaia v9.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed, thanks!


# run full E2E tests in parallel (including multiconsumer) using specific tagged version of gaia
# usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-parallel-tagged
test-gaia-e2e-parallel-tagged:
go run ./tests/e2e/... --include-multi-consumer --parallel --use-gaia --gaia-tag $(GAIA_TAG)
go run ./e2e/... --include-multi-consumer --parallel --use-gaia --gaia-tag $(GAIA_TAG)

# run all tests with caching disabled
test-no-cache:
go test ./... -count=1 && go run ./tests/e2e/...
go test ./... -count=1 && go run ./e2e/...

###############################################################################
### Linting ###
Expand Down
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ brew install jq

**Installing and running binaries**

This repository consists of multiple go modules. In order to have a smooth development experience, please add a `go.work` file. It is enough to copy and rename the `go.work.example` file.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The go.work file is also in this PR, this makes me think it should not. Should we remove the go.work file itself from the PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, the go.work file should not be commited.

It is commited for now because the testing pipelines fail if you don't commit a go work file. The go tool sees that there are 2 modules but does not know how to build and execute normal tests without instructions from go.work.

This can also be fixed by some make magic, e.g.:

# create a go.work file on the fly by renaming go.work.example
mv ./go.work.example ./go.work
go test (...)

I opted to not do that now since it would be more confusing than just commiting the file directly.

Notice how none of the make commands are actually changed, they just point to a different path (./e2e instead of ./testing/e2e).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, makes sense. Then maybe let's change that line in the README to account for the fact that the go.work file is in the repo already and keep it like this for now?


```bash
# install interchain-security-pd and interchain-security-cd binaries
make install
Expand Down
5 changes: 0 additions & 5 deletions cmd/interchain-security-cd/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"

rosettaCmd "cosmossdk.io/tools/rosetta/cmd"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/config"
"github.com/cosmos/cosmos-sdk/client/debug"
Expand Down Expand Up @@ -207,9 +205,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
txCommand(),
keys.Commands(consumer.DefaultNodeHome),
)

// add rosetta
rootCmd.AddCommand(rosettaCmd.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Codec))
}

// newApp is an appCreator
Expand Down
5 changes: 0 additions & 5 deletions cmd/interchain-security-cdd/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"

rosettaCmd "cosmossdk.io/tools/rosetta/cmd"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/config"
"github.com/cosmos/cosmos-sdk/client/debug"
Expand Down Expand Up @@ -207,9 +205,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
txCommand(),
keys.Commands(cdd.DefaultNodeHome),
)

// add rosetta
rootCmd.AddCommand(rosettaCmd.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Codec))
}

// newApp is an appCreator
Expand Down
5 changes: 0 additions & 5 deletions cmd/interchain-security-pd/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"

rosettaCmd "cosmossdk.io/tools/rosetta/cmd"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/config"
"github.com/cosmos/cosmos-sdk/client/debug"
Expand Down Expand Up @@ -207,9 +205,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
txCommand(),
keys.Commands(providerApp.DefaultNodeHome),
)

// add rosetta
rootCmd.AddCommand(rosettaCmd.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Codec))
}

// newApp is an appCreator
Expand Down
5 changes: 0 additions & 5 deletions cmd/interchain-security-sd/cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"

rosettaCmd "cosmossdk.io/tools/rosetta/cmd"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/client/config"
"github.com/cosmos/cosmos-sdk/client/debug"
Expand Down Expand Up @@ -207,9 +205,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
txCommand(),
keys.Commands(sovereignApp.DefaultNodeHome),
)

// add rosetta
rootCmd.AddCommand(rosettaCmd.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Codec))
}

// newApp is an appCreator
Expand Down
2 changes: 1 addition & 1 deletion docs/old/testing.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ To run integration tests against your own consumer/provider implementations, use

## End-to-End (E2E) Tests

[E2E tests](../../tests/e2e/) run true consumer and provider chain binaries within a docker container and are relevant to the highest level of functionality. E2E tests use queries/transactions invoked from CLI to drive and validate the code.
[E2E tests](../../e2e/) run true consumer and provider chain binaries within a docker container and are relevant to the highest level of functionality. E2E tests use queries/transactions invoked from CLI to drive and validate the code.

## Running Tests
Tests can be run using `make`:
Expand Down
43 changes: 18 additions & 25 deletions tests/e2e/actions.go → e2e/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@ import (
"github.com/tidwall/gjson"

evidencetypes "github.com/cosmos/cosmos-sdk/x/evidence/types"

"github.com/cosmos/interchain-security/v3/x/ccv/provider/client"
"github.com/cosmos/interchain-security/v3/x/ccv/provider/types"
ccvtypes "github.com/cosmos/interchain-security/v3/x/ccv/types"
)

type SendTokensAction struct {
Expand Down Expand Up @@ -235,21 +231,20 @@ func (tr TestRun) submitConsumerAdditionProposal(
verbose bool,
) {
spawnTime := tr.containerConfig.now.Add(time.Duration(action.spawnTime) * time.Millisecond)
params := ccvtypes.DefaultParams()
prop := client.ConsumerAdditionProposalJSON{
prop := ConsumerAdditionProposalJSONV3{
Title: "Propose the addition of a new chain",
Summary: "Gonna be a great chain",
ChainId: string(tr.chainConfigs[action.consumerChain].chainId),
InitialHeight: action.initialHeight,
GenesisHash: []byte("gen_hash"),
BinaryHash: []byte("bin_hash"),
SpawnTime: spawnTime,
ConsumerRedistributionFraction: params.ConsumerRedistributionFraction,
BlocksPerDistributionTransmission: params.BlocksPerDistributionTransmission,
HistoricalEntries: params.HistoricalEntries,
CcvTimeoutPeriod: params.CcvTimeoutPeriod,
TransferTimeoutPeriod: params.TransferTimeoutPeriod,
UnbondingPeriod: params.UnbondingPeriod,
ConsumerRedistributionFraction: DefaultCCVParams.ConsumerRedistributionFraction,
BlocksPerDistributionTransmission: DefaultCCVParams.BlocksPerDistributionTransmission,
HistoricalEntries: DefaultCCVParams.HistoricalEntries,
CcvTimeoutPeriod: DefaultCCVParams.CcvTimeoutPeriod,
TransferTimeoutPeriod: DefaultCCVParams.TransferTimeoutPeriod,
UnbondingPeriod: DefaultCCVParams.UnbondingPeriod,
Deposit: fmt.Sprint(action.deposit) + `stake`,
DistributionTransmissionChannel: action.distributionChannel,
}
Expand Down Expand Up @@ -306,7 +301,7 @@ func (tr TestRun) submitConsumerRemovalProposal(
verbose bool,
) {
stopTime := tr.containerConfig.now.Add(action.stopTimeOffset)
prop := client.ConsumerRemovalProposalJSON{
prop := ConsumerRemovalProposalJSONV3{
Title: fmt.Sprintf("Stop the %v chain", action.consumerChain),
Summary: "It was a great chain",
ChainId: string(tr.chainConfigs[action.consumerChain].chainId),
Expand Down Expand Up @@ -442,18 +437,16 @@ func (tr TestRun) submitEquivocationProposal(action submitEquivocationProposalAc
val := tr.validatorConfigs[action.validator]
providerChain := tr.chainConfigs[chainID("provi")]

prop := client.EquivocationProposalJSON{
Summary: "Validator equivocation!",
EquivocationProposal: types.EquivocationProposal{
Title: "Validator equivocation!",
Description: fmt.Sprintf("Validator: %s has committed an equivocation infraction on chainID: %s", action.validator, action.chain),
Equivocations: []*evidencetypes.Equivocation{
{
Height: action.height,
Time: action.time,
Power: action.power,
ConsensusAddress: val.valconsAddress,
},
prop := EquivocationProposalJSONV3{
Summary: "Validator equivocation!",
Title: "Validator equivocation!",
Description: fmt.Sprintf("Validator: %s has committed an equivocation infraction on chainID: %s", action.validator, action.chain),
Equivocations: []*evidencetypes.Equivocation{
{
Height: action.height,
Time: action.time,
Power: action.power,
ConsensusAddress: val.valconsAddress,
},
},
Deposit: fmt.Sprint(action.deposit) + `stake`,
Expand Down
0 tests/e2e/actions_sovereign_chain.go → e2e/actions_sovereign_chain.go
View file
Open in desktop
File renamed without changes.
0 tests/e2e/config.go → e2e/config.go
View file
Open in desktop
File renamed without changes.
Loading
Loading