diff --git a/.gitignore b/.gitignore index 638ecc2e..d232dfc9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ cmd/node/node cmd/node/.b7s_* cmd/node/*.yaml +cmd/b7s-node-docs/b7s-node-docs cmd/keygen/keygen cmd/keyforge/keyforge diff --git a/cmd/b7s-node-docs/assets/css/style.css b/cmd/b7s-node-docs/assets/css/style.css new file mode 100644 index 00000000..ba9c807b --- /dev/null +++ b/cmd/b7s-node-docs/assets/css/style.css @@ -0,0 +1,91 @@ +body { + font-family: Arial, sans-serif; + background-color: #dce4e8; + color: #333; + margin: 20px; + padding: 20px; +} + +h1 { + color: #333; + border-bottom: 1px solid #ccc; + padding-bottom: 10px; + margin-bottom: 20px; +} + +h3 { + color: #666; + margin-top: 20px; +} + +ul { + list-style-type: none; + padding: 0; +} + +li { + margin-bottom: 20px; + padding: 20px; +} + +li.cfg { + background-color: #fff; + border-radius: 15px; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + margin-bottom: 20px; + padding: 20px; +} + +li.child-cfg { + background-color: #fefefe; + border-radius: 15px; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + margin-bottom: 20px; + padding: 20px; +} + +li.cli { + background-color: #edf2f5; + margin-bottom: 10px; + margin-right: 30px; + margin-left: 30px; + padding: 10px; +} + +.cli-details { + margin-left: 20px; +} + +h3 { + color: #333; + margin-top: 0; +} + +p { + margin: 5px 0; +} + +dl { + margin: 0; +} + +dt { + margin-bottom: 15px; +} + +dd { + margin-left: 0; +} + +code { + background-color: #f5f5f5; + padding: 2px 5px; + border-radius: 3px; +} + +.link-icon { + margin-left: 5px; + font-size: 80%; + color: #888; + text-decoration: none; +} diff --git a/cmd/b7s-node-docs/assets/favicon/favicon.ico b/cmd/b7s-node-docs/assets/favicon/favicon.ico new file mode 100644 index 00000000..4d0e1a20 Binary files /dev/null and b/cmd/b7s-node-docs/assets/favicon/favicon.ico differ diff --git a/cmd/b7s-node-docs/b7sdocs.templ b/cmd/b7s-node-docs/b7sdocs.templ new file mode 100644 index 00000000..495449bd --- /dev/null +++ b/cmd/b7s-node-docs/b7sdocs.templ @@ -0,0 +1,86 @@ +package main + +import "fmt" +import "github.com/blocklessnetwork/b7s/config" + +templ page(configs []config.ConfigOption) { + + + Blockless B7S Node Configuration + + + + +

Blockless B7S Node Configuration

+

+ This page lists all of the configuration options supported by the b7s daemon. + It showcases the configuration structure, as accepted in a YAML config file, environment variables that can be used to set those options and, where applicable, the CLI flags and their default values. +

+ + @b7sdocs(configs) + + +} + + +templ b7sdocs(configs []config.ConfigOption) { + + +} + +func formatCLIDefault(def any) string { + str := fmt.Sprint(def) + if str != "" { + return str + } + + return "N/A" +} + +templ configOption(cfg config.ConfigOption) { + +

{cfg.Name} 🔗

+ + if cfg.Type() != "" { +

Type: {cfg.Type()}

+ } + +

Path: {cfg.FullPath}

+ if cfg.Env != "" { +

Environment variable: {cfg.Env}

+ } + + if cfg.CLI.Flag != "" { + +
+
CLI flag:
+
+ +
+
+ } + + if len(cfg.Children) > 0 { + + } +} \ No newline at end of file diff --git a/cmd/b7s-node-docs/b7sdocs_templ.go b/cmd/b7s-node-docs/b7sdocs_templ.go new file mode 100644 index 00000000..0fc04375 --- /dev/null +++ b/cmd/b7s-node-docs/b7sdocs_templ.go @@ -0,0 +1,303 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.2.648 +package main + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import "context" +import "io" +import "bytes" + +import "fmt" +import "github.com/blocklessnetwork/b7s/config" + +func page(configs []config.ConfigOption) templ.Component { + return templ.ComponentFunc(func(ctx context.Context, templ_7745c5c3_W io.Writer) (templ_7745c5c3_Err error) { + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templ_7745c5c3_W.(*bytes.Buffer) + if !templ_7745c5c3_IsBuffer { + templ_7745c5c3_Buffer = templ.GetBuffer() + defer templ.ReleaseBuffer(templ_7745c5c3_Buffer) + } + ctx = templ.InitializeContext(ctx) + templ_7745c5c3_Var1 := templ.GetChildren(ctx) + if templ_7745c5c3_Var1 == nil { + templ_7745c5c3_Var1 = templ.NopComponent + } + ctx = templ.ClearChildren(ctx) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("Blockless B7S Node Configuration

Blockless B7S Node Configuration

This page lists all of the configuration options supported by the b7s daemon. It showcases the configuration structure, as accepted in a YAML config file, environment variables that can be used to set those options and, where applicable, the CLI flags and their default values.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = b7sdocs(configs).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if !templ_7745c5c3_IsBuffer { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteTo(templ_7745c5c3_W) + } + return templ_7745c5c3_Err + }) +} + +func b7sdocs(configs []config.ConfigOption) templ.Component { + return templ.ComponentFunc(func(ctx context.Context, templ_7745c5c3_W io.Writer) (templ_7745c5c3_Err error) { + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templ_7745c5c3_W.(*bytes.Buffer) + if !templ_7745c5c3_IsBuffer { + templ_7745c5c3_Buffer = templ.GetBuffer() + defer templ.ReleaseBuffer(templ_7745c5c3_Buffer) + } + ctx = templ.InitializeContext(ctx) + templ_7745c5c3_Var2 := templ.GetChildren(ctx) + if templ_7745c5c3_Var2 == nil { + templ_7745c5c3_Var2 = templ.NopComponent + } + ctx = templ.ClearChildren(ctx) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if !templ_7745c5c3_IsBuffer { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteTo(templ_7745c5c3_W) + } + return templ_7745c5c3_Err + }) +} + +func formatCLIDefault(def any) string { + str := fmt.Sprint(def) + if str != "" { + return str + } + + return "N/A" +} + +func configOption(cfg config.ConfigOption) templ.Component { + return templ.ComponentFunc(func(ctx context.Context, templ_7745c5c3_W io.Writer) (templ_7745c5c3_Err error) { + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templ_7745c5c3_W.(*bytes.Buffer) + if !templ_7745c5c3_IsBuffer { + templ_7745c5c3_Buffer = templ.GetBuffer() + defer templ.ReleaseBuffer(templ_7745c5c3_Buffer) + } + ctx = templ.InitializeContext(ctx) + templ_7745c5c3_Var3 := templ.GetChildren(ctx) + if templ_7745c5c3_Var3 == nil { + templ_7745c5c3_Var3 = templ.NopComponent + } + ctx = templ.ClearChildren(ctx) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(cfg.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `b7sdocs.templ`, Line: 48, Col: 35} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(" 🔗

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if cfg.Type() != "" { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

Type: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(cfg.Type()) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `b7sdocs.templ`, Line: 51, Col: 28} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

Path: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(cfg.FullPath) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `b7sdocs.templ`, Line: 54, Col: 26} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if cfg.Env != "" { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

Environment variable: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(cfg.Env) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `b7sdocs.templ`, Line: 56, Col: 41} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if cfg.CLI.Flag != "" { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("
CLI flag:
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if len(cfg.Children) > 0 { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if !templ_7745c5c3_IsBuffer { + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteTo(templ_7745c5c3_W) + } + return templ_7745c5c3_Err + }) +} diff --git a/cmd/b7s-node-docs/main.go b/cmd/b7s-node-docs/main.go new file mode 100644 index 00000000..f059edd3 --- /dev/null +++ b/cmd/b7s-node-docs/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "embed" + "fmt" + "log" + "net/http" + "os" + + "github.com/a-h/templ" + "github.com/spf13/pflag" + + "github.com/blocklessnetwork/b7s/config" +) + +//go:embed assets/* +var assets embed.FS + +func main() { + + var ( + flagAddress string + flagOutput string + flagEmbed bool + ) + pflag.StringVarP(&flagAddress, "address", "a", "127.0.0.1:8080", "address to serve on") + pflag.StringVarP(&flagOutput, "output", "o", "", "output file to write the documentation to") + pflag.BoolVarP(&flagEmbed, "embed", "e", true, "use embedded files for assets") + pflag.Parse() + + configs := config.GetConfigDocumentation() + component := page(configs) + + if flagOutput != "" { + + f, err := os.Create(flagOutput) + if err != nil { + log.Fatalf("could not open file: %s", err) + } + + err = component.Render(context.Background(), f) + if err != nil { + log.Fatalf("could not render component: %s", err) + } + + f.Close() + return + } + + mux := http.NewServeMux() + + var fh http.Handler + if flagEmbed { + fh = http.FileServer(http.FS(assets)) + } else { + fh = http.StripPrefix("/assets/", http.FileServer(http.Dir("assets"))) + } + + mux.Handle("/assets/", fh) + mux.Handle("/", templ.Handler(component)) + + fmt.Printf("Documentation served on http://%s/", flagAddress) + + err := http.ListenAndServe(flagAddress, mux) + if err != nil { + log.Fatalf("failed to start server: %s", err) + } +} diff --git a/cmd/node/README.md b/cmd/node/README.md index d7ffce4b..63f5cbe1 100644 --- a/cmd/node/README.md +++ b/cmd/node/README.md @@ -27,7 +27,7 @@ List of supported CLI flags is listed below. ```console Usage of b7s-node: --config string path to a config file - -r, --role string role this note will have in the Blockless protocol (head or worker) (default "worker") + -r, --role string role this node will have in the Blockless protocol (head or worker) (default "worker") -c, --concurrency uint maximum number of requests node will process in parallel (default 10) --boot-nodes strings list of addresses that this node will connect to on startup, in multiaddr format --workspace string directory that the node can use for file storage diff --git a/cmd/node/internal/config/config.go b/cmd/node/internal/config/config.go deleted file mode 100644 index c73930d5..00000000 --- a/cmd/node/internal/config/config.go +++ /dev/null @@ -1,57 +0,0 @@ -package config - -// Config type is tightly coupled with the config options defined in flags.go. -// Flag name should be the same as the value in the `koanf` tag here (flag is `--dialback-address`, the koanf tag is `dialback-address`). -// This is needed so the two ways of loading config are correctly merged. -// -// The `group` of the config option defines in which section of the config file it lives. -// Examples: -// connectivity => address, port, private-key... -// worker => runtime-path, runtime-cli, cpu-percentage-limit... -// - -// Config describes the Blockless configuration options. -type Config struct { - Role string `koanf:"role"` - Concurrency uint `koanf:"concurrency"` - BootNodes []string `koanf:"boot-nodes"` - Workspace string `koanf:"workspace"` // TODO: Check - does a head node ever use a workspace? - LoadAttributes bool `koanf:"attributes"` // TODO: Head node probably doesn't need attributes..? - Topics []string `koanf:"topics"` - - PeerDatabasePath string `koanf:"peer-db"` - FunctionDatabasePath string `koanf:"function-db"` // TODO: Head node doesn't need a function database. - - Log Log `koanf:"log"` - Connectivity Connectivity `koanf:"connectivity"` - Head Head `koanf:"head"` - Worker Worker `koanf:"worker"` -} - -// Log describes the logging configuration. -type Log struct { - Level string `koanf:"level"` -} - -// Connectivity describes the libp2p host that the node will use. -type Connectivity struct { - Address string `koanf:"address"` - Port uint `koanf:"port"` - PrivateKey string `koanf:"private-key"` - DialbackAddress string `koanf:"dialback-address"` - DialbackPort uint `koanf:"dialback-port"` - Websocket bool `koanf:"websocket"` - WebsocketPort uint `koanf:"websocket-port"` - WebsocketDialbackPort uint `koanf:"websocket-dialback-port"` -} - -type Head struct { - API string `koanf:"rest-api"` -} - -type Worker struct { - RuntimePath string `koanf:"runtime-path"` - RuntimeCLI string `koanf:"runtime-cli"` - CPUPercentageLimit float64 `koanf:"cpu-percentage-limit"` - MemoryLimitKB int64 `koanf:"memory-limit"` -} diff --git a/cmd/node/internal/config/flags.go b/cmd/node/internal/config/flags.go deleted file mode 100644 index 9000430d..00000000 --- a/cmd/node/internal/config/flags.go +++ /dev/null @@ -1,213 +0,0 @@ -package config - -import ( - "github.com/blocklessnetwork/b7s/node" - "github.com/spf13/pflag" -) - -// Default values. -const ( - DefaultPort = uint(0) - DefaultAddress = "0.0.0.0" - DefaultRole = "worker" - DefaultPeerDB = "peer-db" - DefaultFunctionDB = "function-db" - DefaultConcurrency = uint(node.DefaultConcurrency) - DefaultUseWebsocket = false - DefaultWorkspace = "workspace" -) - -type configOption struct { - flag string // long flag name - should be the same as the `koanf` tag in the Config type. - short string // shorthand - single letter alternative to the long flag name - group configGroup // group - defined in which section of the config file this option lives. - usage string // description -} - -// Config options. -var ( - // Root group. - roleCfg = configOption{ - flag: "role", - short: "r", - group: rootGroup, - usage: "role this note will have in the Blockless protocol (head or worker)", - } - concurrencyCfg = configOption{ - flag: "concurrency", - short: "c", - group: rootGroup, - usage: "maximum number of requests node will process in parallel", - } - bootNodesCfg = configOption{ - flag: "boot-nodes", - group: rootGroup, - usage: "list of addresses that this node will connect to on startup, in multiaddr format", - } - workspaceCfg = configOption{ - flag: "workspace", - group: rootGroup, - usage: "directory that the node can use for file storage", - } - attributesCfg = configOption{ - flag: "attributes", - group: rootGroup, - usage: "node should try to load its attribute data from IPFS", - } - peerDBCfg = configOption{ - flag: "peer-db", - group: rootGroup, - usage: "path to the database used for persisting peer data", - } - functionDBCfg = configOption{ - flag: "function-db", - group: rootGroup, - usage: "path to the database used for persisting function data", - } - topicsCfg = configOption{ - flag: "topics", - group: rootGroup, - usage: "topics node should subscribe to", - } - - // Log group. - logLevelCfg = configOption{ - flag: "log-level", - short: "l", - group: logGroup, - usage: "log level to use", - } - - // Connectivity group. - addressCfg = configOption{ - flag: "address", - short: "a", - group: connectivityGroup, - usage: "address that the b7s host will use", - } - portCfg = configOption{ - flag: "port", - short: "p", - group: connectivityGroup, - usage: "port that the b7s host will use", - } - privateKeyCfg = configOption{ - flag: "private-key", - group: connectivityGroup, - usage: "private key that the b7s host will use", - } - websocketCfg = configOption{ - flag: "websocket", - short: "w", - group: connectivityGroup, - usage: "should the node use websocket protocol for communication", - } - websocketPortCfg = configOption{ - flag: "websocket-port", - group: connectivityGroup, - usage: "port to use for websocket connections", - } - dialbackAddressCfg = configOption{ - flag: "dialback-address", - group: connectivityGroup, - usage: "external address that the b7s host will advertise", - } - dialbackPortCfg = configOption{ - flag: "dialback-port", - group: connectivityGroup, - usage: "external port that the b7s host will advertise", - } - websocketDialbackPortCfg = configOption{ - flag: "websocket-dialback-port", - group: connectivityGroup, - usage: "external port that the b7s host will advertise for websocket connections", - } - - // Worker flags. - runtimePathCfg = configOption{ - flag: "runtime-path", - group: workerGroup, - usage: "Blockless Runtime location (used by the worker node)", - } - runtimeCLICfg = configOption{ - flag: "runtime-cli", - group: workerGroup, - usage: "runtime CLI name (used by the worker node)", - } - cpuLimitCfg = configOption{ - flag: "cpu-percentage-limit", - group: workerGroup, - usage: "amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited", - } - memLimitCfg = configOption{ - flag: "memory-limit", - group: workerGroup, - usage: "memory limit (kB) for Blockless Functions", - } - - // Head node flags. - restAPICfg = configOption{ - flag: "rest-api", - group: headGroup, - usage: "address where the head node REST API will listen on", - } -) - -// This helper type is a thin wrapper around the pflag.FlagSet. -// Added functionality is the accounting of added flags. -// This is needed/useful when we're translating flags between the structured format (yaml file) and the flat structure (CLI flags). -type cliFlags struct { - fs *pflag.FlagSet - options []configOption -} - -func newCliFlags() *cliFlags { - - fs := pflag.NewFlagSet("b7s-node", pflag.ExitOnError) - fs.SortFlags = false - - return &cliFlags{ - fs: fs, - options: make([]configOption, 0), - } -} - -func (c *cliFlags) stringFlag(cfg configOption, defaultValue string) { - c.fs.StringP(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) boolFlag(cfg configOption, defaultValue bool) { - c.fs.BoolP(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) uintFlag(cfg configOption, defaultValue uint) { - c.fs.UintP(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) int64Flag(cfg configOption, defaultValue int64) { - c.fs.Int64P(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) float64Flag(cfg configOption, defaultValue float64) { - c.fs.Float64P(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) stringSliceFlag(cfg configOption, defaultValue []string) { - c.fs.StringSliceP(cfg.flag, cfg.short, defaultValue, cfg.usage) - c.options = append(c.options, cfg) -} - -func (c *cliFlags) groups() map[string]configGroup { - - groups := make(map[string]configGroup) - for _, option := range c.options { - groups[option.flag] = option.group - } - - return groups -} diff --git a/cmd/node/internal/config/load.go b/cmd/node/internal/config/load.go deleted file mode 100644 index 89c8c039..00000000 --- a/cmd/node/internal/config/load.go +++ /dev/null @@ -1,117 +0,0 @@ -package config - -import ( - "fmt" - "os" - - "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/providers/posflag" - "github.com/knadh/koanf/v2" - "github.com/spf13/pflag" - - "github.com/blocklessnetwork/b7s/models/blockless" -) - -func Load(args ...string) (*Config, error) { - return load(os.Args[1:]) -} - -func load(args []string) (*Config, error) { - - var configPath string - - flags := newCliFlags() - flags.fs.StringVar(&configPath, "config", "", "path to a config file") - - // General flags. - flags.stringFlag(roleCfg, DefaultRole) - flags.uintFlag(concurrencyCfg, DefaultConcurrency) - flags.stringSliceFlag(bootNodesCfg, nil) - flags.stringFlag(workspaceCfg, "") - flags.boolFlag(attributesCfg, false) - flags.stringFlag(peerDBCfg, "") - flags.stringFlag(functionDBCfg, "") - flags.stringSliceFlag(topicsCfg, nil) - - // Log. - flags.stringFlag(logLevelCfg, "info") - - // Connectivity flags. - flags.stringFlag(addressCfg, DefaultAddress) - flags.uintFlag(portCfg, DefaultPort) - flags.stringFlag(privateKeyCfg, "") - flags.boolFlag(websocketCfg, DefaultUseWebsocket) - flags.uintFlag(websocketPortCfg, DefaultPort) - flags.stringFlag(dialbackAddressCfg, DefaultAddress) - flags.uintFlag(dialbackPortCfg, DefaultPort) - flags.uintFlag(websocketDialbackPortCfg, DefaultPort) - - // Worker node flags. - flags.stringFlag(runtimePathCfg, "") - flags.stringFlag(runtimeCLICfg, blockless.RuntimeCLI()) - flags.float64Flag(cpuLimitCfg, 1) - flags.int64Flag(memLimitCfg, 0) - - // Head node flags. - flags.stringFlag(restAPICfg, "") - - flags.fs.Parse(args) - - delimiter := "." - konfig := koanf.New(delimiter) - - if configPath != "" { - err := konfig.Load(file.Provider(configPath), yaml.Parser()) - if err != nil { - return nil, fmt.Errorf("could not load config file: %w", err) - } - } - - // For readability flags have a flat structure - e.g. port or cpu-percentage-limit. - // For use in config files, we prefer a structured layout, e.g. connectivity=>port or worker=>cpu-percentage-limit. - // This callback translates the flag names from a flat layout to the structured one, so that koanf knows how to match - // analogous values. - translate := flagTranslate(flags.groups(), flags.fs, delimiter) - - err := konfig.Load(posflag.ProviderWithFlag(flags.fs, delimiter, konfig, translate), nil) - if err != nil { - return nil, fmt.Errorf("could not load config: %w", err) - } - - var cfg Config - err = konfig.Unmarshal("", &cfg) - if err != nil { - return nil, fmt.Errorf("could not unmarshal konfig: %w", err) - } - - return &cfg, nil -} - -func flagTranslate(flagGroups map[string]configGroup, fs *pflag.FlagSet, delimiter string) func(*pflag.Flag) (string, any) { - - return func(flag *pflag.Flag) (string, any) { - key := flag.Name - val := posflag.FlagVal(fs, flag) - - // Should not happen. - group, ok := flagGroups[key] - if !ok { - return key, val - } - - name := group.Name() - if name == "" { - return key, val - } - - // Log level is a special case because the CLI flag is already prefixed (--log-level). - if key == logLevelCfg.flag { - skey := "log" + delimiter + "level" - return skey, val - } - - skey := name + delimiter + key - return skey, val - } -} diff --git a/cmd/node/main.go b/cmd/node/main.go index adbb83ee..6d7ab3d6 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -15,7 +15,7 @@ import ( "github.com/ziflex/lecho/v3" "github.com/blocklessnetwork/b7s/api" - "github.com/blocklessnetwork/b7s/cmd/node/internal/config" + "github.com/blocklessnetwork/b7s/config" "github.com/blocklessnetwork/b7s/executor" "github.com/blocklessnetwork/b7s/executor/limits" "github.com/blocklessnetwork/b7s/fstore" @@ -90,8 +90,8 @@ func run() int { log.Info(). Str("workspace", cfg.Workspace). - Str("peer_db", cfg.PeerDatabasePath). - Str("function_db", cfg.FunctionDatabasePath). + Str("peer_db", cfg.PeerDB). + Str("function_db", cfg.FunctionDB). Msg("filepaths used by the node") // Convert workspace path to an absolute one. @@ -103,9 +103,9 @@ func run() int { cfg.Workspace = workspace // Open the pebble peer database. - pdb, err := pebble.Open(cfg.PeerDatabasePath, &pebble.Options{Logger: &pebbleNoopLogger{}}) + pdb, err := pebble.Open(cfg.PeerDB, &pebble.Options{Logger: &pebbleNoopLogger{}}) if err != nil { - log.Error().Err(err).Str("db", cfg.PeerDatabasePath).Msg("could not open pebble peer database") + log.Error().Err(err).Str("db", cfg.PeerDB).Msg("could not open pebble peer database") return failure } defer pdb.Close() @@ -203,9 +203,9 @@ func run() int { } // Open the pebble function database. - fdb, err := pebble.Open(cfg.FunctionDatabasePath, &pebble.Options{Logger: &pebbleNoopLogger{}}) + fdb, err := pebble.Open(cfg.FunctionDB, &pebble.Options{Logger: &pebbleNoopLogger{}}) if err != nil { - log.Error().Err(err).Str("db", cfg.FunctionDatabasePath).Msg("could not open pebble function database") + log.Error().Err(err).Str("db", cfg.FunctionDB).Msg("could not open pebble function database") return failure } defer fdb.Close() @@ -255,7 +255,7 @@ func run() int { // If we're a head node - start the REST API. if role == blockless.HeadNode { - if cfg.Head.API == "" { + if cfg.Head.RestAPI == "" { log.Error().Err(err).Msg("REST API address is required") return failure } @@ -281,8 +281,8 @@ func run() int { // Start API in a separate goroutine. go func() { - log.Info().Str("port", cfg.Head.API).Msg("Node API starting") - err := server.Start(cfg.Head.API) + log.Info().Str("port", cfg.Head.RestAPI).Msg("Node API starting") + err := server.Start(cfg.Head.RestAPI) if err != nil && !errors.Is(err, http.ErrServerClosed) { log.Warn().Err(err).Msg("Node API failed") close(failed) @@ -326,17 +326,17 @@ func updateDirPaths(root string, cfg *config.Config) { } cfg.Workspace = workspace - peerDB := cfg.PeerDatabasePath + peerDB := cfg.PeerDB if peerDB == "" { peerDB = filepath.Join(root, config.DefaultPeerDB) } - cfg.PeerDatabasePath = peerDB + cfg.PeerDB = peerDB - functionDB := cfg.FunctionDatabasePath + functionDB := cfg.FunctionDB if functionDB == "" { functionDB = filepath.Join(root, config.DefaultFunctionDB) } - cfg.FunctionDatabasePath = functionDB + cfg.FunctionDB = functionDB } func generateNodeDirName(id string) string { diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..aeddbe8e --- /dev/null +++ b/config/config.go @@ -0,0 +1,145 @@ +package config + +import ( + "github.com/blocklessnetwork/b7s/node" +) + +// Default values. +const ( + DefaultPort = uint(0) + DefaultAddress = "0.0.0.0" + DefaultRole = "worker" + DefaultPeerDB = "peer-db" + DefaultFunctionDB = "function-db" + DefaultConcurrency = uint(node.DefaultConcurrency) + DefaultUseWebsocket = false + DefaultWorkspace = "" + DefaultLogLevel = "info" +) + +var DefaultConfig = Config{ + Role: DefaultRole, + Concurrency: DefaultConcurrency, + PeerDB: DefaultPeerDB, + FunctionDB: DefaultFunctionDB, + Workspace: DefaultWorkspace, + Log: Log{ + Level: DefaultLogLevel, + }, + Connectivity: Connectivity{ + Address: DefaultAddress, + Port: DefaultPort, + Websocket: DefaultUseWebsocket, + }, +} + +// Config describes the Blockless configuration options. +// NOTE: DO NOT use TABS in struct tags - spaces only! +// NOTE: When adding CLI flags (using the `flag` struct tag) - add the description for (for the flag long version, not the shorthand) it in getFlagDescription() below. +type Config struct { + Role string `koanf:"role" flag:"role,r"` + Concurrency uint `koanf:"concurrency" flag:"concurrency,c"` + BootNodes []string `koanf:"boot-nodes" flag:"boot-nodes"` + Workspace string `koanf:"workspace" flag:"workspace"` // TODO: Check - does a head node ever use a workspace? + LoadAttributes bool `koanf:"load-attributes" flag:"load-attributes"` // TODO: Head node probably doesn't need attributes..? + Topics []string `koanf:"topics" flag:"topics"` + + PeerDB string `koanf:"peer-db" flag:"peer-db"` + FunctionDB string `koanf:"function-db" flag:"function-db"` // TODO: Head node doesn't need a function database. + + Log Log `koanf:"log"` + Connectivity Connectivity `koanf:"connectivity"` + Head Head `koanf:"head"` + Worker Worker `koanf:"worker"` +} + +// Log describes the logging configuration. +type Log struct { + Level string `koanf:"level" flag:"log-level,l"` +} + +// Connectivity describes the libp2p host that the node will use. +type Connectivity struct { + Address string `koanf:"address" flag:"address,a"` + Port uint `koanf:"port" flag:"port,p"` + PrivateKey string `koanf:"private-key" flag:"private-key"` + DialbackAddress string `koanf:"dialback-address" flag:"dialback-address"` + DialbackPort uint `koanf:"dialback-port" flag:"dialback-port"` + Websocket bool `koanf:"websocket" flag:"websocket,w"` + WebsocketPort uint `koanf:"websocket-port" flag:"websocket-port"` + WebsocketDialbackPort uint `koanf:"websocket-dialback-port" flag:"websocket-dialback-port"` +} + +type Head struct { + RestAPI string `koanf:"rest-api" flag:"rest-api"` +} + +type Worker struct { + RuntimePath string `koanf:"runtime-path" flag:"runtime-path"` + RuntimeCLI string `koanf:"runtime-cli" flag:"runtime-cli"` + CPUPercentageLimit float64 `koanf:"cpu-percentage-limit" flag:"cpu-percentage-limit"` + MemoryLimitKB int64 `koanf:"memory-limit" flag:"memory-limit"` +} + +// ConfigOptionInfo describes a specific configuration option, it's location in the config file and +// corresponding CLI flags and environment variables. It can be used to generate documentation for the b7s node. +type ConfigOptionInfo struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + FullPath string `json:"full_path,omitempty" yaml:"full_path,omitempty"` + CLI CLIFlag `json:"cli,omitempty" yaml:"cli,omitempty"` + Env string `json:"env-var,omitempty" yaml:"env-var,omitempty"` + Children []ConfigOption `json:"children,omitempty" yaml:"children,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` +} + +func getFlagDescription(flag string) string { + + switch flag { + case "role": + return "role this node will have in the Blockless protocol (head or worker)" + case "concurrency": + return "maximum number of requests node will process in parallel" + case "boot-nodes": + return "list of addresses that this node will connect to on startup, in multiaddr format" + case "workspace": + return "directory that the node can use for file storage" + case "load-attributes": + return "node should try to load its attribute data from IPFS" + case "topics": + return "topics node should subscribe to" + case "peer-db": + return "path to the database used for persisting peer data" + case "function-db": + return "path to the database used for persisting function data" + case "log-level": + return "log level to use" + case "address": + return "address that the b7s host will use" + case "port": + return "port that the b7s host will use" + case "private-key": + return "private key that the b7s host will use" + case "websocket": + return "should the node use websocket protocol for communication" + case "dialback-address": + return "external address that the b7s host will advertise" + case "dialback-port": + return "external port that the b7s host will advertise" + case "websocket-port": + return "port to use for websocket connections" + case "websocket-dialback-port": + return "external port that the b7s host will advertise for websocket connections" + case "rest-api": + return "address where the head node REST API will listen on" + case "runtime-path": + return "Blockless Runtime location (used by the worker node)" + case "runtime-cli": + return "runtime CLI name (used by the worker node)" + case "cpu-percentage-limit": + return "amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited" + case "memory-limit": + return "memory limit (kB) for Blockless Functions" + default: + return "" + } +} diff --git a/config/config_options.go b/config/config_options.go new file mode 100644 index 00000000..d2c90c79 --- /dev/null +++ b/config/config_options.go @@ -0,0 +1,136 @@ +package config + +import ( + "fmt" + "reflect" + "strings" + + "github.com/blocklessnetwork/b7s/models/blockless" +) + +type ConfigOption struct { + Name string `yaml:"name,omitempty"` + FullPath string `yaml:"full_path,omitempty"` + CLI CLIFlag `yaml:"cli,omitempty"` + Env string `yaml:"env-var,omitempty"` + Children []ConfigOption `yaml:"children,omitempty"` + + kind reflect.Kind + elementKind reflect.Kind // if kind is a slice, tell us what the elements of the slice are. +} + +func (c ConfigOption) Type() string { + + // For slices say something like "list (string)", for primitive types print the type, and skip structs. + if c.kind == reflect.Slice { + return fmt.Sprintf("list (%s)", c.elementKind.String()) + } + + if c.kind != reflect.Struct { + return c.kind.String() + } + + return "" +} + +func (c ConfigOption) Info() ConfigOptionInfo { + + info := ConfigOptionInfo{ + FullPath: c.FullPath, + Name: c.Name, + CLI: c.CLI, + Env: c.Env, + Children: c.Children, + Type: c.Type(), + } + + return info +} + +type CLIFlag struct { + Flag string `yaml:"flag,omitempty"` + Shorthand string `yaml:"shorthand,omitempty"` + Default any `yaml:"default,omitempty"` + Description string `yaml:"description,omitempty"` +} + +func getConfigOptions() []ConfigOption { + cliDefaults := getDefaultFlagValues() + return getStructInfo(reflect.TypeOf(Config{}), cliDefaults) +} + +func getStructInfo(typ reflect.Type, cliDefaults map[string]any, parents ...string) []ConfigOption { + + out := make([]ConfigOption, 0) + for _, field := range reflect.VisibleFields(typ) { + + var ( + kind = field.Type.Kind() + koanfTag = field.Tag.Get("koanf") + parts = fullPath(koanfTag, parents...) + fullPath = strings.Join(parts, ".") + ) + + fi := ConfigOption{ + FullPath: fullPath, + kind: kind, + Name: koanfTag, + // Env variable is set later, after we determine the type + } + + ft := field.Tag.Get("flag") + if ft != "" { + flag, shorthand := getFlagFromTag(ft) + + cli := CLIFlag{ + Flag: flag, + Shorthand: shorthand, + Default: cliDefaults[fullPath], + Description: getFlagDescription(flag), + } + + fi.CLI = cli + } + + switch kind { + + case reflect.Struct: + children := getStructInfo(field.Type, cliDefaults, parts...) + fi.Children = children + + case reflect.Slice, reflect.Array: + fi.elementKind = field.Type.Elem().Kind() + fi.Env = envName(koanfTag, parents...) + + default: + fi.Env = envName(koanfTag, parents...) + } + + out = append(out, fi) + } + + return out +} + +func envName(name string, parents ...string) string { + + parts := make([]string, 0) + for i := len(parents) - 1; i >= 0; i-- { + title := strings.Title(parents[i]) + parts = append(parts, title) + } + + nameFields := strings.Split(name, "-") + var formattedName string + for _, field := range nameFields { + titled := strings.Title(field) + formattedName += titled + } + + var components []string + components = append(components, strings.TrimSuffix(blockless.EnvPrefix, EnvDelimiter)) // Trim trailing underscore so we don't repeat it. + components = append(components, parts...) + components = append(components, formattedName) + + return strings.Join(components, EnvDelimiter) +} diff --git a/config/documentation.go b/config/documentation.go new file mode 100644 index 00000000..dfa1bf39 --- /dev/null +++ b/config/documentation.go @@ -0,0 +1,5 @@ +package config + +func GetConfigDocumentation() []ConfigOption { + return getConfigOptions() +} diff --git a/config/flags.go b/config/flags.go new file mode 100644 index 00000000..b091c4a5 --- /dev/null +++ b/config/flags.go @@ -0,0 +1,105 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + "github.com/knadh/koanf/providers/structs" + "github.com/spf13/pflag" +) + +func createFlags(fields []ConfigOption) (*pflag.FlagSet, map[string]string, error) { + + fs := pflag.NewFlagSet("b7s-node", pflag.ExitOnError) + fs.SortFlags = false + + for _, field := range fields { + err := addFlag(fs, field.CLI) + if err != nil { + return nil, nil, fmt.Errorf("could not add flag for config (name: %v, flag: %v, type: %v)", field.FullPath, field.CLI.Flag, field.kind.String()) + } + } + + mapping, err := mapCLIFlagsToConfig(fields) + if err != nil { + return nil, nil, fmt.Errorf("could not get mapping of CLI flags to config: %w", err) + } + + return fs, mapping, nil +} + +func addFlag(fs *pflag.FlagSet, fc CLIFlag) error { + + if fc.Flag == "" { + return nil + } + + switch def := fc.Default.(type) { + case uint: + fs.UintP(fc.Flag, fc.Shorthand, def, fc.Description) + + case string: + fs.StringP(fc.Flag, fc.Shorthand, def, fc.Description) + + case float64: + fs.Float64P(fc.Flag, fc.Shorthand, def, fc.Description) + + case int64: + fs.Int64P(fc.Flag, fc.Shorthand, def, fc.Description) + + case bool: + fs.BoolP(fc.Flag, fc.Shorthand, def, fc.Description) + + case []string: + fs.StringSliceP(fc.Flag, fc.Shorthand, nil, fc.Description) + + default: + return errors.New("unsupported type for a CLI flag. Extend support by adding handling for the new flag type") + } + + return nil +} + +func getFlagFromTag(tag string) (string, string) { + + tag = strings.TrimSpace(tag) + + fields := strings.Split(tag, ",") + switch len(fields) { + case 0: + return "", "" + case 1: + return fields[0], "" + default: + return fields[0], fields[1] + } +} + +// return mapping of CLI flag to the config path used by koanf. E.g. address => connectivity.address. +// We don't have to enfore uniqueness of CLI flags as pflag does that for us. +func mapCLIFlagsToConfig(fields []ConfigOption) (map[string]string, error) { + + flags := make(map[string]string) + for _, field := range fields { + if field.CLI.Flag == "" { + continue + } + flags[field.CLI.Flag] = field.FullPath + } + + return flags, nil +} + +func getDefaultFlagValues() map[string]any { + + cfg := structs.Provider(DefaultConfig, "koanf") + defaults, err := cfg.Read() + if err != nil { + return nil + } + + flat := make(map[string]any) + flattenMap("", defaults, flat) + return flat +} diff --git a/cmd/node/internal/config/group.go b/config/group.go similarity index 100% rename from cmd/node/internal/config/group.go rename to config/group.go diff --git a/config/helpers.go b/config/helpers.go new file mode 100644 index 00000000..67c50f88 --- /dev/null +++ b/config/helpers.go @@ -0,0 +1,43 @@ +package config + +// By default we have flags in a tree structure, but most of the time we just want the leaves. +func flattenConfigOptions(cfgOptions []ConfigOption) []ConfigOption { + out := make([]ConfigOption, 0, len(cfgOptions)) + for _, cfg := range cfgOptions { + expandConfigOption(cfg, &out) + } + return out +} + +func expandConfigOption(cfg ConfigOption, out *[]ConfigOption) { + if len(cfg.Children) == 0 { + *out = append(*out, cfg) + return + } + for _, fc := range cfg.Children { + expandConfigOption(fc, out) + } +} + +func flattenMap(prefix string, in map[string]any, flat map[string]any) { + for k, v := range in { + key := k + if prefix != "" { + key = prefix + "." + k + } + switch cv := v.(type) { + default: + flat[key] = v + + case map[string]any: + flattenMap(key, cv, flat) + } + } +} + +func fullPath(name string, parents ...string) []string { + var full []string + full = append(full, parents...) + full = append(full, name) + return full +} diff --git a/config/helpers_test.go b/config/helpers_test.go new file mode 100644 index 00000000..5cd14da9 --- /dev/null +++ b/config/helpers_test.go @@ -0,0 +1,40 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFlattenMap(t *testing.T) { + + in := map[string]any{ + "k1": "v1", + "k2": "v2", + "k3": map[string]any{ + "k3-1": "v3-1", + "k3-2": "v3-2", + "k3-3": map[string]any{ + "k3-3-1": "v3-3-1", + "k3-3-2": "v3-3-2", + }, + }, + "k4": map[string]any{ + "k4-1": map[string]any{ + "k4-1-1": "v4-1-1", + }, + }, + } + + flat := make(map[string]any) + flattenMap("", in, flat) + + require.Len(t, flat, 7) + require.Equal(t, flat["k1"], "v1") + require.Equal(t, flat["k2"], "v2") + require.Equal(t, flat["k3.k3-1"], "v3-1") + require.Equal(t, flat["k3.k3-2"], "v3-2") + require.Equal(t, flat["k3.k3-3.k3-3-1"], "v3-3-1") + require.Equal(t, flat["k3.k3-3.k3-3-2"], "v3-3-2") + require.Equal(t, flat["k4.k4-1.k4-1-1"], "v4-1-1") +} diff --git a/config/load.go b/config/load.go new file mode 100644 index 00000000..fe62e638 --- /dev/null +++ b/config/load.go @@ -0,0 +1,124 @@ +package config + +import ( + "fmt" + "os" + "strings" + + "github.com/fatih/camelcase" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/env" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" + "github.com/knadh/koanf/v2" + "github.com/spf13/pflag" + + "github.com/blocklessnetwork/b7s/models/blockless" +) + +const ( + defaultDelimiter = "." + EnvDelimiter = "_" +) + +func Load() (*Config, error) { + return load(os.Args[1:]) +} + +func load(args []string) (*Config, error) { + + var configPath string + + configOptions := flattenConfigOptions(getConfigOptions()) + flags, mapping, err := createFlags(configOptions) + if err != nil { + return nil, fmt.Errorf("could not create CLI flags for config: %w", err) + } + + flags.StringVar(&configPath, "config", "", "path to a config file") + + // General flags. + flags.Parse(args) + + delimiter := defaultDelimiter + konfig := koanf.New(delimiter) + + err = konfig.Load(env.ProviderWithValue(blockless.EnvPrefix, EnvDelimiter, envClean), nil) + if err != nil { + return nil, fmt.Errorf("could not load configuration from env: %w", err) + } + + if configPath != "" { + err = konfig.Load(file.Provider(configPath), yaml.Parser()) + if err != nil { + return nil, fmt.Errorf("could not load config file: %w", err) + } + } + + // For the sake of usability flags have a flat structure - e.g. port or cpu-percentage-limit. + // For use in config files, we prefer a structured layout, e.g. connectivity=>port or worker=>cpu-percentage-limit. + // This callback translates the flag names from a flat layout to the structured one, so that koanf knows how to match + // analogous values. + translate := cliFlagTranslate(mapping, flags) + + err = konfig.Load(posflag.ProviderWithFlag(flags, delimiter, konfig, translate), nil) + if err != nil { + return nil, fmt.Errorf("could not load config: %w", err) + } + + var cfg Config + err = konfig.Unmarshal("", &cfg) + if err != nil { + return nil, fmt.Errorf("could not unmarshal konfig: %w", err) + } + + return &cfg, nil +} + +func cliFlagTranslate(mapping map[string]string, fs *pflag.FlagSet) func(*pflag.Flag) (string, any) { + + return func(flag *pflag.Flag) (string, any) { + key := flag.Name + val := posflag.FlagVal(fs, flag) + + // Should not happen. + skey, ok := mapping[key] + if !ok { + return key, val + } + + return skey, val + } +} + +// envClean will do the following: +// +// - split environment variables to parts ("B7S_Connectivity_DialbackAddress" => [ "Connectivity", "DialbackAddress"]) +// - translate individual parts from CamelCase to Kebab-Case ("DialbackAddress" => "Dialback-Address") +// - lowercase parts ("Dialback-Address" => "dialback-address") +// - join back parts using the environment variable delimiter (underscore) ("Connectivity_DialbackAddress" => "connectivity_dialback-address") +// +// Koanf then uses the underscore to determine structure and in which section the config option belongs. +func envClean(key string, value string) (string, any) { + + key = strings.TrimPrefix(key, blockless.EnvPrefix) + + sections := strings.Split(key, EnvDelimiter) + cleaned := make([]string, 0, len(sections)) + for _, part := range sections { + p := strings.ToLower(strings.Join(camelcase.Split(part), "-")) + cleaned = append(cleaned, p) + } + + ss := strings.Join(cleaned, EnvDelimiter) + + switch ss { + + default: + return ss, value + + // Kludge: For boot nodes and topics, return type should be a string slice. + case "boot-nodes", "topics": + return ss, strings.Split(value, ",") + } +} diff --git a/cmd/node/internal/config/load_test.go b/config/load_test.go similarity index 56% rename from cmd/node/internal/config/load_test.go rename to config/load_test.go index 7957c810..4b7edfd7 100644 --- a/cmd/node/internal/config/load_test.go +++ b/config/load_test.go @@ -173,8 +173,9 @@ func TestConfig_CLIArgsWithConfigFile(t *testing.T) { filepath := writeConfigFile(t, cfgMap) + // NOTE: For compatiblity with Windows we will manually append config param later because `shlex.Split` doesn't jive with Windows paths. cmdline := fmt.Sprintf( - "--role %v --runtime-path %v --concurrency %v --workspace %v --boot-nodes %v --log-level %v --address %v --port %v --cpu-percentage-limit %v --rest-api %v --config %v", + "--role %v --runtime-path %v --concurrency %v --workspace %v --boot-nodes %v --log-level %v --address %v --port %v --cpu-percentage-limit %v --rest-api %v", role, runtimePathCLI, concurrencyCLI, @@ -185,12 +186,13 @@ func TestConfig_CLIArgsWithConfigFile(t *testing.T) { portCLI, cpuPercentageLimitCLI, restAPICLI, - filepath, ) args, err := shlex.Split(cmdline) require.NoError(t, err) + args = append(args, "--config", fmt.Sprintf("%v", filepath)) + cfg, err := load(args) require.NoError(t, err) @@ -210,7 +212,153 @@ func TestConfig_CLIArgsWithConfigFile(t *testing.T) { require.Equal(t, runtimePathCLI, cfg.Worker.RuntimePath) require.Equal(t, websocketFile, cfg.Connectivity.Websocket) require.Equal(t, websocketPortFile, cfg.Connectivity.WebsocketPort) - require.Equal(t, restAPICLI, cfg.Head.API) + require.Equal(t, restAPICLI, cfg.Head.RestAPI) +} + +func TestConfig_Environment(t *testing.T) { + + const ( + role = "worker" + concurrency = uint(45) + bootNodes = "a,b,c,d" + topics = "topic1,topic2,topic3" + + peerDB = "/tmp/db/peer-db" + functionDB = "/tmp/db/function-db" + + logLevel = "trace" + + address = "127.0.0.1" + port = uint(9000) + dialbackPort = uint(9001) + websocket = true + websocketPort = uint(10000) + websocketDialbackPort = uint(10001) + + runtimePath = "/tmp/runtime" + cpuPercentageLimit = float64(0.97) + memoryLimit = int64(512_000) + ) + + t.Setenv("B7S_Role", role) + t.Setenv("B7S_Concurrency", fmt.Sprint(concurrency)) + t.Setenv("B7S_BootNodes", bootNodes) + t.Setenv("B7S_Topics", topics) + t.Setenv("B7S_PeerDB", peerDB) + t.Setenv("B7S_FunctionDB", functionDB) + t.Setenv("B7S_Log_Level", logLevel) + t.Setenv("B7S_Connectivity_Address", address) + t.Setenv("B7S_Connectivity_Port", fmt.Sprint(port)) + t.Setenv("B7S_Connectivity_DialbackPort", fmt.Sprint(dialbackPort)) + t.Setenv("B7S_Connectivity_Websocket", fmt.Sprint(websocket)) + t.Setenv("B7S_Connectivity_WebsocketPort", fmt.Sprint(websocketPort)) + t.Setenv("B7S_Connectivity_WebsocketDialbackPort", fmt.Sprint(websocketDialbackPort)) + t.Setenv("B7S_Worker_RuntimePath", runtimePath) + t.Setenv("B7S_Worker_CPUPercentageLimit", fmt.Sprint(cpuPercentageLimit)) + t.Setenv("B7S_Worker_MemoryLimit", fmt.Sprint(memoryLimit)) + + cfg, err := Load() + require.NoError(t, err) + + require.Equal(t, role, cfg.Role) + require.Equal(t, concurrency, cfg.Concurrency) + + nodeList := strings.Split(bootNodes, ",") + require.Equal(t, nodeList, cfg.BootNodes) + + topicList := strings.Split(topics, ",") + require.Equal(t, topicList, cfg.Topics) + + require.Equal(t, peerDB, cfg.PeerDB) + require.Equal(t, functionDB, cfg.FunctionDB) + require.Equal(t, logLevel, cfg.Log.Level) + require.Equal(t, address, cfg.Connectivity.Address) + require.Equal(t, port, cfg.Connectivity.Port) + require.Equal(t, dialbackPort, cfg.Connectivity.DialbackPort) + require.Equal(t, websocket, cfg.Connectivity.Websocket) + require.Equal(t, websocketPort, cfg.Connectivity.WebsocketPort) + require.Equal(t, websocketDialbackPort, cfg.Connectivity.WebsocketDialbackPort) + + require.Equal(t, runtimePath, cfg.Worker.RuntimePath) + require.Equal(t, cpuPercentageLimit, cfg.Worker.CPUPercentageLimit) + require.Equal(t, memoryLimit, cfg.Worker.MemoryLimitKB) +} + +func TestConfig_Priority(t *testing.T) { + + const ( + envWorkspace = "/tmp/env/workspace" + envAddress = "1.1.1.1" + envPort = uint(1) + envRuntimePath = "/tmp/env/runtime/path" + envLogLevel = "error" + + cfgWorkspace = "/tmp/cfg/workspace" + cfgAddress = "2.2.2.2" + cfgPort = uint(2) + cfgDialbackPort = uint(12) + + cliWorkspace = "/tmp/cli/workspace" + cliAddress = "3.3.3.3" + cliLogLevel = "debug" + ) + + var ( + cfgMap = map[string]any{ + "workspace": cfgWorkspace, + "connectivity": map[string]any{ + "address": cfgAddress, + "port": cfgPort, + "dialback-port": cfgDialbackPort, + }, + } + ) + + filepath := writeConfigFile(t, cfgMap) + + t.Setenv("B7S_Workspace", envWorkspace) + t.Setenv("B7S_Connectivity_Address", envAddress) + t.Setenv("B7S_Connectivity_Port", fmt.Sprint(envPort)) + t.Setenv("B7S_Worker_RuntimePath", envRuntimePath) + t.Setenv("B7S_Log_Level", envLogLevel) + + // NOTE: For compatiblity with Windows we will manually append config param later because `shlex.Split` doesn't jive with Windows paths. + cmdline := fmt.Sprintf( + "--workspace %v --address %v --log-level %v", + cliWorkspace, + cliAddress, + cliLogLevel, + ) + + args, err := shlex.Split(cmdline) + require.NoError(t, err) + + args = append(args, "--config", fmt.Sprintf("%v", filepath)) + + cfg, err := load(args) + require.NoError(t, err) + + // Verify resulting config. + // + // 1. CLI flags override everything + // 2. Config file overrides environment variables + // 3. Environment variables + // + // Any config option set via lower priority methods persists if it's not overwritten. + + // This is set only via env. + require.Equal(t, envRuntimePath, cfg.Worker.RuntimePath) + + // This is set in config file and not overwritten by CLI flags, so it should remain active. + require.Equal(t, cfgPort, cfg.Connectivity.Port) + // This is only set in config file. + require.Equal(t, cfgDialbackPort, cfg.Connectivity.DialbackPort) + + // CLI flags rule everything. + require.Equal(t, cliWorkspace, cfg.Workspace) + require.Equal(t, cliAddress, cfg.Connectivity.Address) + require.Equal(t, cliLogLevel, cfg.Log.Level) + } func writeConfigFile(t *testing.T, m map[string]any) string { diff --git a/go.mod b/go.mod index 5cad79a8..7e3d6c72 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/cavaliergopher/grab/v3 v3.0.1 github.com/cockroachdb/pebble v1.0.0 github.com/containerd/cgroups/v3 v3.0.3 + github.com/fatih/camelcase v1.0.0 github.com/fatih/color v1.16.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/go-hclog v1.3.0 @@ -16,8 +17,10 @@ require ( github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/ipfs/boxo v0.17.0 github.com/knadh/koanf/parsers/yaml v0.1.0 + github.com/knadh/koanf/providers/env v0.1.0 github.com/knadh/koanf/providers/file v0.1.0 github.com/knadh/koanf/providers/posflag v0.1.0 + github.com/knadh/koanf/providers/structs v0.1.0 github.com/knadh/koanf/v2 v2.1.0 github.com/labstack/echo/v4 v4.11.4 github.com/libp2p/go-libp2p v0.33.2 @@ -33,11 +36,13 @@ require ( ) require ( + github.com/a-h/templ v0.2.648 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.26.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index ec1937e8..d0ca8f88 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/a-h/templ v0.2.648 h1:A1ggHGIE7AONOHrFaDTM8SrqgqHL6fWgWCijQ21Zy9I= +github.com/a-h/templ v0.2.648/go.mod h1:SA7mtYwVEajbIXFRh3vKdYm/4FYyLQAtPH1+KxzGPA8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -93,9 +95,13 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -104,8 +110,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -264,10 +270,14 @@ github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NI github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY= +github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg= +github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ= github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c= github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA= github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U= github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0= +github.com/knadh/koanf/providers/structs v0.1.0 h1:wJRteCNn1qvLtE5h8KQBvLJovidSdntfdyIbbCzEyE0= +github.com/knadh/koanf/providers/structs v0.1.0/go.mod h1:sw2YZ3txUcqA3Z27gPlmmBzWn1h8Nt9O6EP/91MkcWE= github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8= github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -669,7 +679,6 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/models/blockless/errors.go b/models/blockless/params.go similarity index 68% rename from models/blockless/errors.go rename to models/blockless/params.go index 6ee070d3..bda19e06 100644 --- a/models/blockless/errors.go +++ b/models/blockless/params.go @@ -2,6 +2,8 @@ package blockless import ( "errors" + + "github.com/libp2p/go-libp2p/core/protocol" ) // Sentinel errors. @@ -10,3 +12,8 @@ var ( ErrRollCallTimeout = errors.New("roll call timed out - not enough nodes responded") ErrExecutionNotEnoughNodes = errors.New("not enough execution results received") ) + +const ( + ProtocolID protocol.ID = "/b7s/work/1.0.0" + EnvPrefix string = "B7S_" +) diff --git a/models/blockless/protocol.go b/models/blockless/protocol.go deleted file mode 100644 index 7976e2e5..00000000 --- a/models/blockless/protocol.go +++ /dev/null @@ -1,9 +0,0 @@ -package blockless - -import ( - "github.com/libp2p/go-libp2p/core/protocol" -) - -const ( - ProtocolID protocol.ID = "/b7s/work/1.0.0" -)