Skip to content

Commit

Permalink
Bringup docker based integration tests
Browse files Browse the repository at this point in the history 
This gets the docker based env up and running with the dns resolved hostnames.

`task dev:testnet:up` works.

Started working on tests to deploy namespaces, tables and actions. Will get to reviving the rest of the tests in other PR.
  • Loading branch information
@charithabandi
charithabandi authored Jan 6, 2025
1 parent cdc6878 commit d9b806f
Show file tree
Hide file tree
Showing 13 changed files with 442 additions and 324 deletions.
66 changes: 33 additions & 33 deletions Taskfile.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,42 +144,42 @@ tasks:
docker volume rm -f {{ .NAME }}
# ************ dev ************
# dev:up:
# desc: Start the dev environment
# deps:
# - task: build:docker
# cmds:
# - task: dev:up:nb

# dev:up:debug:
# desc: Start the dev environment
# deps:
# - task: build:docker
# vars: { VARIANT: 'debug' }
# cmds:
# - task: dev:up:nb
dev:up:
desc: Start the dev environment
deps:
- task: build:docker
cmds:
- task: dev:up:nb

# dev:up:nb:
# desc: Start the dev environment without rebuilding docker image
# env:
# # NOTE: this timeout should be long enough to attach to debugger
# KACT_WAIT_TIMEOUT: 20s
# dir: test # different module
# cmds:
# - go test ./acceptance -run ^TestLocalDevSetup -timeout 12h -dev -v {{.CLI_ARGS}}
dev:up:debug:
desc: Start the dev environment
deps:
- task: build:docker
vars: { VARIANT: 'debug' }
cmds:
- task: dev:up:nb

dev:up:nb:
desc: Start the dev environment without rebuilding docker image
env:
# NOTE: this timeout should be long enough to attach to debugger
KACT_WAIT_TIMEOUT: 20s
dir: test # different module
cmds:
- go test ./acceptance -run ^TestLocalDevSetup -timeout 12h -dev -v {{.CLI_ARGS}}

# dev:testnet:up:
# desc: Start the dev environment(with testnet)
# deps:
# - task: build:docker
# cmds:
# - task: dev:testnet:up:nb
dev:testnet:up:
desc: Start the dev environment(with testnet)
deps:
- task: build:docker
cmds:
- task: dev:testnet:up:nb

# dev:testnet:up:nb:
# desc: Start the dev environment(with testnet) without rebuilding docker image
# dir: test # different module
# cmds:
# - go test ./integration -run ^TestLocalDevSetup$ -timeout 12h -dev -v {{.CLI_ARGS}}
dev:testnet:up:nb:
desc: Start the dev environment(with testnet) without rebuilding docker image
dir: test # different module
cmds:
- go test ./integration -run ^TestLocalDevSetup$ -timeout 12h -dev -v {{.CLI_ARGS}}

# ************ test ************
# test with build:docker task support passing CLI_ARGS to go test, e.g. task test:act -- -debug
Expand Down
108 changes: 84 additions & 24 deletions app/setup/testnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,24 @@ import (

func TestnetCmd() *cobra.Command {
var numVals, numNVals int
var noPex bool
var noPex, uniquePorts bool
var startingPort uint64
var outDir string

cmd := &cobra.Command{
Use: "testnet",
Short: "Generate configuration for multiple nodes",
RunE: func(cmd *cobra.Command, args []string) error {
return GenerateTestnetConfigs(outDir, numVals, numNVals, noPex, startingPort)
return GenerateTestnetConfigs(&TestnetConfig{
RootDir: outDir,
NumVals: numVals,
NumNVals: numNVals,
NoPex: noPex,
StartingPort: startingPort,
}, &ConfigOpts{
UniquePorts: uniquePorts,
DnsHost: false,
})
},
}

Expand All @@ -50,14 +59,42 @@ func TestnetCmd() *cobra.Command {
cmd.Flags().BoolVar(&noPex, "no-pex", false, "disable peer exchange")
cmd.Flags().Uint64VarP(&startingPort, "port", "p", 6600, "starting P2P port for the nodes")
cmd.Flags().StringVarP(&outDir, "out-dir", "o", ".testnet", "output directory for generated node root directories")

cmd.Flags().BoolVarP(&uniquePorts, "unique-ports", "u", false, "use unique ports for each node")
return cmd
}

func GenerateTestnetConfigs(outDir string, numVals, numNVals int, noPex bool, startingPort uint64) error {
type TestnetConfig struct {
RootDir string
ChainID string
NumVals int
NumNVals int
NoPex bool
StartingPort uint64
StartingIP string
HostnamePrefix string
DnsNamePrefix string // optional and only used if DnsHost is true (default: node)

Owner string
}

type ConfigOpts struct {
// UniquePorts is a flag to generate unique listening addresses
// (JSON-RPC, HTTP, Admin, P2P, node RPC) for each node.
// This is useful for testing multiple nodes on the same machine.
// If it is used for generating a single config, it has no effect.
UniquePorts bool

// DnsHost is a flag to use DNS hostname as host in the config
// instead of ip. It will be used together with DnsNamePrefix to generate
// hostnames.
// This is useful for testing nodes inside docker containers.
DnsHost bool
}

func GenerateTestnetConfigs(cfg *TestnetConfig, opts *ConfigOpts) error {
// ensure that the directory exists
// expand the directory path
outDir, err := node.ExpandPath(outDir)
outDir, err := node.ExpandPath(cfg.RootDir)
if err != nil {
return err
}
Expand All @@ -68,10 +105,10 @@ func GenerateTestnetConfigs(outDir string, numVals, numNVals int, noPex bool, st

var keys []crypto.PrivateKey
// generate the configuration for the nodes
for i := range numVals + numNVals {
for i := range cfg.NumVals + cfg.NumNVals {
// generate Keys, so that the connection strings and the validator set can be generated before the node config files are generated
var seed [32]byte
binary.LittleEndian.PutUint64(seed[:], startingPort+uint64(i))
binary.LittleEndian.PutUint64(seed[:], cfg.StartingPort+uint64(i))
seed = sha256.Sum256(seed[:])
rr := rand.NewChaCha8(seed)
priv := node.NewKey(&deterministicPRNG{ChaCha8: rr})
Expand All @@ -81,33 +118,64 @@ func GenerateTestnetConfigs(outDir string, numVals, numNVals int, noPex bool, st
// key 0 is leader
leaderPub := keys[0].Public()

var bootNodes []string
for i := range cfg.NumVals {
pubKey := keys[i].Public()

hostname := cfg.StartingIP
if cfg.StartingIP == "" {
hostname = "127.0.0.1"
}

if opts.DnsHost {
hostname = fmt.Sprintf("%s%d", cfg.DnsNamePrefix, i)
}

port := 6600
if opts.UniquePorts {
port = 6600 + i
}

bootNodes = append(bootNodes, node.FormatPeerString(pubKey.Bytes(), pubKey.Type(), hostname, port))
}

chainID := cfg.ChainID
if chainID == "" {
chainID = "kwil-testnet"
}
genConfig := &config.GenesisConfig{
ChainID: "kwil-testnet",
ChainID: chainID,
Leader: leaderPub.Bytes(), // rethink this so it can be different key types?
Validators: make([]*ktypes.Validator, numVals),
Validators: make([]*ktypes.Validator, cfg.NumVals),
DisabledGasCosts: true,
JoinExpiry: 14400,
VoteExpiry: 108000,
MaxBlockSize: 6 * 1024 * 1024,
MaxVotesPerTx: 200,
DBOwner: cfg.Owner,
}

for i := range numVals {
for i := range cfg.NumVals {
genConfig.Validators[i] = &ktypes.Validator{
PubKey: keys[i].Public().Bytes(),
Power: 1,
}
}

// generate the configuration for the nodes
for i := range numVals + numNVals {
portOffset := 0
for i := range cfg.NumVals + cfg.NumNVals {
if opts.UniquePorts {
portOffset = i
}
err = GenerateNodeRoot(&NodeGenConfig{
PortOffset: i,
IP: "127.0.0.1",
NoPEX: noPex,
PortOffset: portOffset,
IP: cfg.StartingIP,
NoPEX: cfg.NoPex,
RootDir: filepath.Join(outDir, fmt.Sprintf("node%d", i)),
NodeKey: keys[i],
Genesis: genConfig,
BootNodes: bootNodes,
})
if err != nil {
return err
Expand All @@ -127,6 +195,7 @@ type NodeGenConfig struct {
Genesis *config.GenesisConfig

// TODO: gasEnabled, private p2p, auth RPC, join expiry, allocs, etc.
BootNodes []string
}

func GenerateNodeRoot(ncfg *NodeGenConfig) error {
Expand All @@ -144,16 +213,7 @@ func GenerateNodeRoot(ncfg *NodeGenConfig) error {
}
cfg.P2P.Pex = !ncfg.NoPEX

leaderPub, err := crypto.UnmarshalPublicKey(ncfg.Genesis.Leader, crypto.KeyTypeSecp256k1)
if err != nil {
return err
}

if !ncfg.NodeKey.Public().Equals(leaderPub) {
// make everyone connect to leader
cfg.P2P.BootNodes = []string{node.FormatPeerString(
leaderPub.Bytes(), leaderPub.Type(), cfg.P2P.IP, 6600)}
}
cfg.P2P.BootNodes = ncfg.BootNodes

// DB
dbPort := ncfg.DBPort
Expand Down
9 changes: 6 additions & 3 deletions node/consensus/leader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,17 +177,20 @@ func (ce *ConsensusEngine) addVote(ctx context.Context, vote *vote, sender strin
defer ce.state.mtx.Unlock()

if ce.state.blkProp == nil {
return errors.New("not processing any block proposal at the moment")
ce.log.Warn("Error adding vote: not processing any block proposal at the moment")
return nil
}

// check if the vote is for the current height
if ce.state.blkProp.height != vote.height {
return errors.New("vote received for a different block height, ignore it")
ce.log.Warn("Error adding vote: Vote received for a different block height, ignore it", "height", vote.height)
return nil
}

// check if the vote is for the current block and from a validator
if ce.state.blkProp.blkHash != vote.blkHash {
return fmt.Errorf("vote received for an incorrect block %s", vote.blkHash.String())
ce.log.Warn("Error adding vote: Vote received for a different block", "height", vote.height, "blkHash", vote.blkHash)
return nil
}

// Check if the vote is from a validator
Expand Down
16 changes: 16 additions & 0 deletions test/driver/cli_driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,22 @@ func (d *KwilCliDriver) Execute(ctx context.Context, dbid string, action string,
return out.TxHash, nil
}

func (d *KwilCliDriver) ExecuteSQL(ctx context.Context, sql string, params map[string]any) (types.Hash, error) {
// actionInputs, err := d.prepareCliActionParams(ctx, dbid, action, inputs[0])
// if err != nil {
// return types.Hash{}, fmt.Errorf("failed to get action params: %w", err)
// }

args := []string{"database", "execute", "--sql", sql}

cmd := d.newKwilCliCmd(args...)
out, err := mustRun[respTxHash](cmd, d.logger)
if err != nil {
return types.Hash{}, fmt.Errorf("failed to execute action: %w", err)
}
return out.TxHash, nil
}

func (d *KwilCliDriver) QueryDatabase(_ context.Context, query string) (*types.QueryResult, error) {
args := []string{"database", "query", query}

Expand Down
8 changes: 8 additions & 0 deletions test/driver/client_driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,14 @@ func (d *KwildClientDriver) Execute(ctx context.Context, dbid string, actionName
return rec, nil
}

func (d *KwildClientDriver) ExecuteSQL(ctx context.Context, sql string, params map[string]any) (types.Hash, error) {
rec, err := d.clt.ExecuteSQL(ctx, sql, params)
if err != nil {
return types.Hash{}, fmt.Errorf("error executing sql statement %s: error: %w", sql, err)
}
return rec, nil
}

func (d *KwildClientDriver) QueryDatabase(ctx context.Context, query string) (*types.QueryResult, error) {
return d.clt.Query(ctx, query, nil)
}
Expand Down
17 changes: 8 additions & 9 deletions test/integration/docker-compose-dev.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: "3"

services:
# TODO: generate correspond number of nodes and exts by configuration
# TODO: generate corresponding number of nodes and exts by configuration
node0:
image: kwild:latest
ports:
Expand All @@ -24,14 +24,13 @@ services:
--root=/app/kwil
--log-format=plain
--log-level=${LOG_LEVEL:-info}
--app.extension-endpoints=ext1:50051
--app.admin-listen-addr=/tmp/admin.socket
--chain.p2p.listen-addr=tcp://0.0.0.0:26656
--chain.rpc.listen-addr=tcp://0.0.0.0:26657
--app.pg-db-host=pg0
--app.pg-db-port=5432
--app.pg-db-user=kwild
--app.pg-db-pass=kwild
--admin.listen=/tmp/admin.socket
--p2p.ip=tcp:0.0.0.0
--p2p.port=6600
--db.host=pg
--db.port=5432
--db.user=kwild
--db.pass=kwild
healthcheck:
test: ["CMD", "curl", "--fail-with-body", "-s", "http://127.0.0.1:8484/api/v1/health/user"]
interval: 2s
Expand Down
Loading

0 comments on commit d9b806f

Please sign in to comment.