Serialization and other minor fixes, updated docs for setup commands

app/node/build.go

@@ -212,12 +212,7 @@ func restoreDB(d *coreDependencies) bool {
 	}
 
 	// Snapshot file exists
-	genFileName, err := node.ExpandPath(appCfg.GenesisState)
-	if err != nil {
-		failBuild(err, "failed to expand genesis state path")
-	}
-
-	snapFile, err := os.Open(genFileName)
+	snapFile, err := os.Open(appCfg.GenesisState)
 	if err != nil {
 		failBuild(err, "failed to open genesis state file")
 	}
@@ -381,7 +376,7 @@ func buildMigrator(d *coreDependencies, db *pg.DB, accounts *accounts.Accounts,
 		RecurringHeight: d.cfg.Snapshots.RecurringHeight,
 		Enable:          d.cfg.Snapshots.Enable,
 		DBConfig:        &d.cfg.DB,
-	}, d.logger.New("SNAP"))
+	}, d.namespaceManager, d.logger.New("SNAP"))
 	if err != nil {
 		failBuild(err, "failed to create migration's snapshot store")
 	}
@@ -502,7 +497,7 @@ func buildSnapshotStore(d *coreDependencies) *snapshotter.SnapshotStore {
 		failBuild(err, "failed to create snapshot directory")
 	}
 
-	ss, err := snapshotter.NewSnapshotStore(cfg, d.logger.New("SNAP"))
+	ss, err := snapshotter.NewSnapshotStore(cfg, d.namespaceManager, d.logger.New("SNAP"))
 	if err != nil {
 		failBuild(err, "failed to create snapshot store")
 	}

app/node/namespace_manager.go

@@ -1,10 +1,13 @@
 package node
 
 import (
+	"context"
+	"fmt"
 	"sync"
 
 	"github.com/kwilteam/kwil-db/core/utils/order"
 	"github.com/kwilteam/kwil-db/node/engine"
+	"github.com/kwilteam/kwil-db/node/types/sql"
 )
 
 func newNamespaceManager() *namespaceManager {
@@ -87,3 +90,40 @@ func (n *namespaceManager) ListPostgresSchemasToDump() []string {
 
 	return res
 }
+
+type DB interface {
+	sql.ReadTxMaker
+}
+
+func UserNamespaces(ctx context.Context, db DB) ([]string, error) {
+	tx, err := db.BeginReadTx(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback(ctx)
+
+	res, err := tx.Execute(ctx, "select name from kwild_engine.namespaces where type='USER'")
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute query: %w", err)
+	}
+
+	if len(res.Columns) != 1 {
+		return nil, fmt.Errorf("unexpected number of columns: %d", len(res.Columns))
+	}
+
+	var userNamespaces []string
+	for _, row := range res.Rows {
+		if len(row) != 1 {
+			return nil, fmt.Errorf("unexpected number of columns in row: %d", len(row))
+		}
+
+		ns, ok := row[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("failed to convert namespace to string: %v", row[0])
+		}
+
+		userNamespaces = append(userNamespaces, ns)
+	}
+
+	return userNamespaces, nil
+}

app/node/node.go

@@ -124,6 +124,10 @@ func runNode(ctx context.Context, rootDir string, cfg *config.Config, autogen bo
 		return fmt.Errorf("genesis configuration failed sanity checks: %w", err)
 	}
 
+	if cfg.GenesisState != "" {
+		cfg.GenesisState = rootedPath(cfg.GenesisState, rootDir)
+	}
+
 	// if running in autogen mode, and config.toml does not exist, write it
 	tomlFile := config.ConfigFilePath(rootDir)
 	if autogen && !fileExists(tomlFile) {

app/setup/genesis-hash.go

@@ -89,7 +89,7 @@ func GenesisHashCmd() *cobra.Command {
 				}
 				defer cleanupTmpKwilAdminDir(dir) // clean up temp admin snapshots directory on exit after app hash computation
 
-				_, _, genCfg, err := snapshot.PGDump(cmd.Context(), pgConf.DBName, pgConf.User, pgConf.Pass, pgConf.Host, pgConf.Port, dir)
+				_, _, _, genCfg, err := snapshot.PGDump(cmd.Context(), pgConf.DBName, pgConf.User, pgConf.Pass, pgConf.Host, pgConf.Port, dir)
 				if err != nil {
 					return display.PrintErr(cmd, err)
 				}

app/setup/genesis.go

@@ -167,29 +167,43 @@ func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *
 				return nil, fmt.Errorf("invalid format for alloc, expected id#keyType:balance, received: %s", a)
 			}
 
+			// 0x addresses: <address>:<balance>
+			// others: <pubkey#keyType>:<balance>
 			keyParts := strings.Split(parts[0], "#")
-			if len(keyParts) != 2 {
-				return nil, fmt.Errorf("invalid address for alloc: %s", parts[0])
-			}
-
-			keyType, err := crypto.ParseKeyType(keyParts[1])
-			if err != nil {
-				return nil, fmt.Errorf("invalid key type for validator: %s", keyParts[1])
+			var keyType crypto.KeyType
+			var keyStr string
+			var err error
+
+			if strings.HasPrefix(parts[1], "0x") {
+				if len(keyParts) != 1 {
+					return nil, fmt.Errorf("invalid address for alloc: %s, expected format <address:balance>", parts[0])
+				}
+				keyStr = strings.TrimPrefix(parts[0], "0x")
+				keyType = crypto.KeyTypeSecp256k1
+			} else {
+				if len(keyParts) != 2 {
+					return nil, fmt.Errorf("invalid address for alloc: %s, expected format <key#keyType:balance>", parts[0])
+				}
+				keyType, err = crypto.ParseKeyType(keyParts[1])
+				if err != nil {
+					return nil, fmt.Errorf("invalid key type for validator: %s", keyParts[1])
+				}
+				keyStr = keyParts[0]
 			}
 
 			balance, ok := new(big.Int).SetString(parts[1], 10)
 			if !ok {
 				return nil, fmt.Errorf("invalid balance for alloc: %s", parts[1])
 			}
 
-			keyStr, err := hex.DecodeString(keyParts[0])
+			id, err := hex.DecodeString(keyStr)
 			if err != nil {
-				return nil, fmt.Errorf("invalid address for alloc: %s", keyParts[0])
+				return nil, fmt.Errorf("invalid hex address for alloc: %s", keyStr)
 			}
 
 			conf.Allocs = append(conf.Allocs, config.GenesisAlloc{
 				ID: config.KeyHexBytes{
-					HexBytes: keyStr,
+					HexBytes: id,
 				},
 				KeyType: keyType.String(),
 				Amount:  balance,

app/setup/init.go

@@ -39,7 +39,13 @@ This permits rapid prototyping and evaluation of Kwil functionality. An output d
 If no output directory is specified, the node will be initialized ` + "`./testnet`" + `.`
 
 	initExample = `# Initialize a node, with a new network, in the directory ~/.kwil-new
-kwild setup init -r ~/.kwild-new`
+kwild setup init -r ~/.kwild-new
+
+# Run the init command with --allocs flag to initialize a node with initial account balances. 
+The allocs flag should be a comma-separated list of <id#keyType:amount> pairs where the id is the account address or the pubkey and keyType is either "secp256k1" or "ed25519". If id is the ethereum address prefixed with "0x", the keyType is optional as shown below.
+
+kwild setup init --allocs "0xc89D42189f0450C2b2c3c61f58Ec5d628176A1E7:10000000000000000000000,0226b3ff29216dac187cea393f8af685ad419ac9644e55dce83d145c8b1af213bd#secp256k1:56000000000" -r ~/.kwild
+`
 )
 
 func InitCmd() *cobra.Command {
@@ -146,14 +152,19 @@ func InitCmd() *cobra.Command {
 					return display.PrintErr(cmd, fmt.Errorf("failed to create genesis file: %w", err))
 				}
 
-				genCfg.Leader = types.PublicKey{PublicKey: privKey.Public()}
-				genCfg.Validators = append(genCfg.Validators, &types.Validator{
-					AccountID: types.AccountID{
-						Identifier: privKey.Public().Bytes(),
-						KeyType:    privKey.Type(),
-					},
-					Power: 1,
-				})
+				if !cmd.Flags().Changed("leader") {
+					genCfg.Leader = types.PublicKey{PublicKey: privKey.Public()}
+				}
+
+				if len(genCfg.Validators) == 0 {
+					genCfg.Validators = append(genCfg.Validators, &types.Validator{
+						AccountID: types.AccountID{
+							Identifier: privKey.Public().Bytes(),
+							KeyType:    privKey.Type(),
+						},
+						Power: 1,
+					})
+				}
 
 				// If DB owner is not set, set it to the node's public key
 				if genCfg.DBOwner == "" {
@@ -189,6 +200,8 @@ func InitCmd() *cobra.Command {
 				return err
 			}
 
+			fmt.Println("Kwil node configuration generated successfully at: ", outDir)
+
 			return nil
 		},
 	}

app/setup/testnet.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/types"
 	ktypes "github.com/kwilteam/kwil-db/core/types"
+	"github.com/kwilteam/kwil-db/core/utils"
 	authExt "github.com/kwilteam/kwil-db/extensions/auth"
 	"github.com/kwilteam/kwil-db/node"
 )
@@ -242,7 +243,7 @@ func GenerateNodeRoot(ncfg *NodeGenConfig) error {
 	// Admin RPC
 	cfg.Admin.ListenAddress = net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(8584+ncfg.PortOffset), 10))
 
-	return GenerateNodeDir(ncfg.RootDir, ncfg.Genesis, cfg, ncfg.NodeKey)
+	return GenerateNodeDir(ncfg.RootDir, ncfg.Genesis, cfg, ncfg.NodeKey, "")
 }
 
 type deterministicPRNG struct {
@@ -286,13 +287,14 @@ type TestnetNodeConfig struct {
 // GenerateTestnetDir generates a testnet configuration for multiple nodes.
 // It is a minimal function that takes full configurations.
 // Most users should use GenerateTestnetConfigs instead.
-func GenerateTestnetDir(testnetDir string, genesis *config.GenesisConfig, nodes []*TestnetNodeConfig) error {
+// copies the snapshot file to each node directory if it is not empty.
+func GenerateTestnetDir(testnetDir string, genesis *config.GenesisConfig, nodes []*TestnetNodeConfig, snapshot string) error {
 	if err := os.MkdirAll(testnetDir, 0755); err != nil {
 		return err
 	}
 
 	for _, node := range nodes {
-		if err := GenerateNodeDir(filepath.Join(testnetDir, node.DirName), genesis, node.Config, node.PrivateKey); err != nil {
+		if err := GenerateNodeDir(filepath.Join(testnetDir, node.DirName), genesis, node.Config, node.PrivateKey, snapshot); err != nil {
 			return err
 		}
 	}
@@ -303,11 +305,18 @@ func GenerateTestnetDir(testnetDir string, genesis *config.GenesisConfig, nodes
 // GenerateNodeDir generates a node configuration directory.
 // It is a minimal function that takes a full configuration.
 // Most users should use GenerateNodeRoot instead.
-func GenerateNodeDir(rootDir string, genesis *config.GenesisConfig, node *config.Config, privateKey *crypto.Secp256k1PrivateKey) error {
+func GenerateNodeDir(rootDir string, genesis *config.GenesisConfig, node *config.Config, privateKey *crypto.Secp256k1PrivateKey, snapshot string) error {
 	if err := os.MkdirAll(rootDir, 0755); err != nil {
 		return err
 	}
 
+	if snapshot != "" {
+		node.GenesisState = "genesis-state.sql.gz"
+		if err := utils.CopyFile(snapshot, config.GenesisStateFileName(rootDir)); err != nil {
+			return err
+		}
+	}
+
 	if err := node.SaveAs(config.ConfigFilePath(rootDir)); err != nil {
 		return err
 	}