From 581ef8bd2d5875a1f47fca2de59ebf9a324ffed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ljubi=C5=A1a=20Ga=C4=8Devi=C4=87?= <35105035+gacevicljubisa@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:23:16 +0100 Subject: [PATCH] feat: add PostageTTL and PostageLabel options to all Beekeeper checks (#454) * feat: enable PostageLabel option for all checks creating batches * refactor: use geth-url as global flag * feat(stamper): introduce postage-label as filter * fix(stamper): log price in bzz * feat(swap): add offset option for FetchBlockTime * feat: replace postage-amount with postage-ttl * chore: remove unused code * chore: bump version to 0.26.0 * chore: go mod tidy * fix: resolve deps * chore: update configs * chore: update configs * fix: set amount to 1000 if price is 0 * fix(stake): set geth-url * fix(redundancy): set PostageLabel type to string * feat: automatically set semantic version number * fix(swap): set options offset default to 1 * fix(swap): improve FetchBlockTime if offset is too large * chore(swap): add log msg when offset too large * fix(swap): limit offset to at most of half of the latest block * fix(swap): increase offset to 1000 * fix: improve error msgs and flag descriptions * fix(orchestration): ensure that SwapClient is initialized * fix(stamper): ensure that SwapClient is initialized --- Makefile | 5 +- README.md | 7 +- cmd/beekeeper/cmd/check.go | 6 +- cmd/beekeeper/cmd/cluster.go | 7 +- cmd/beekeeper/cmd/cmd.go | 32 +++--- cmd/beekeeper/cmd/create_bee_cluster.go | 8 +- cmd/beekeeper/cmd/fund.go | 2 - cmd/beekeeper/cmd/node_funder.go | 28 +++-- cmd/beekeeper/cmd/operator.go | 21 ++-- cmd/beekeeper/cmd/stamper.go | 36 ++++--- config/config.yaml | 98 ++++++++--------- config/local.yaml | 71 ++++++------ config/public-testnet.yaml | 46 ++++---- config/staging.yaml | 5 +- config/testnet-giant.yaml | 6 -- pkg/bee/api/postage.go | 21 ++-- pkg/bee/client.go | 47 ++++++-- pkg/check/act/act.go | 30 +++--- pkg/check/balances/balances.go | 17 ++- pkg/check/feed/feed.go | 20 ++-- pkg/check/fileretrieval/fileretrieval.go | 8 +- pkg/check/gsoc/gsoc.go | 18 ++-- pkg/check/manifest/manifest.go | 8 +- pkg/check/networkavailability/check.go | 8 +- pkg/check/pss/pss.go | 8 +- pkg/check/pullsync/pullsync.go | 6 +- pkg/check/pushsync/check_chunks.go | 2 +- pkg/check/pushsync/check_lightnode.go | 4 +- pkg/check/pushsync/pushsync.go | 6 +- pkg/check/redundancy/redundancy.go | 20 ++-- pkg/check/retrieval/retrieval.go | 6 +- pkg/check/settlements/settlements.go | 6 +- pkg/check/smoke/load.go | 4 +- pkg/check/smoke/smoke.go | 8 +- pkg/check/soc/soc.go | 6 +- pkg/check/stake/contractutil.go | 7 +- pkg/check/stake/stake.go | 2 - pkg/config/check.go | 131 +++++++++++++---------- pkg/orchestration/cluster.go | 4 +- pkg/orchestration/k8s/cluster.go | 31 +----- pkg/orchestration/k8s/nodegroup.go | 11 +- pkg/orchestration/nodegroup.go | 2 - pkg/simulate/retrieval/retrieval.go | 6 +- pkg/simulate/upload/upload.go | 6 +- pkg/stamper/node.go | 58 ++++++---- pkg/stamper/stamper.go | 47 ++++++-- pkg/swap/block.go | 84 +++++++++++++-- pkg/swap/notset.go | 4 +- pkg/swap/swap.go | 2 +- pkg/test/case.go | 15 +-- pkg/test/node.go | 4 +- version.go | 5 +- 52 files changed, 597 insertions(+), 453 deletions(-) diff --git a/Makefile b/Makefile index 809fb13ba..c5390c7fe 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,10 @@ GO ?= go GOLANGCI_LINT ?= golangci-lint GOLANGCI_LINT_VERSION ?= v1.61.0 COMMIT ?= "$(shell git describe --long --dirty --always --match "" || true)" -LDFLAGS ?= -s -w -X github.com/ethersphere/beekeeper.commit=$(COMMIT) +VERSION ?= "$(shell git describe --tags --abbrev=0 | cut -c2-)" +LDFLAGS ?= -s -w \ +-X github.com/ethersphere/beekeeper.commit="$(COMMIT)" \ +-X github.com/ethersphere/beekeeper.version="$(VERSION)" .PHONY: all all: build lint vet test-race binary diff --git a/README.md b/README.md index e6e9d49b0..97bc24d78 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,6 @@ It has following flags: ```console --cluster-name string cluster name (default "default") ---geth-url string Endpoint to chain node. Required. --help help for bee-cluster --timeout duration timeout (default 30m0s) --wallet-key string Hex-encoded private key for the Bee node wallet. Required. @@ -307,7 +306,6 @@ It has the following flags: --bzz-token-address string BZZ token address (default "0x6aab14fe9cccd64a502d23842d916eb5321c26e7") --eth-account string ETH account address (default "0x62cab2b3b55f341f10348720ca18063cdb779ad5") --eth-deposit float ETH amount to deposit ---geth-url string Geth node URL (default "http://geth-swap.geth-swap.dai.internal") --help help for fund --password password for generating Ethereum addresses (default "beekeeper") --timeout duration timeout (default 5m0s) @@ -381,7 +379,6 @@ It has following flags: ```console --addresses strings Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name. --cluster-name string Name of the Beekeeper cluster to target. Ignored if a namespace is specified. ---geth-url string Endpoint to chain node. Required. --help help for node-funder --label-selector string Kubernetes label selector for filtering resources within the specified namespace. Use an empty string to select all resources. (default "beekeeper.ethswarm.org/node-funder=true") --min-native float Minimum amount of chain native coins (xDAI) nodes should have. @@ -417,7 +414,6 @@ Command **node-operator** uses tool It has following flags: ```console ---geth-url string Endpoint to chain node. Required. --help help for node-operator --min-native float Minimum amount of chain native coins (xDAI) nodes should have. --min-swarm float Minimum amount of swarm tokens (xBZZ) nodes should have. @@ -507,7 +503,6 @@ It has following subcommands: ```console --batch-ids strings Comma separated list of postage batch IDs to top up. If not provided, all batches are topped up. --cluster-name string Target Beekeeper cluster name. - --geth-url string Geth URL for chain state retrieval. --help help for topup --label-selector string Kubernetes label selector for filtering resources (use empty string for all). (default "beekeeper.ethswarm.org/node-funder=true") --namespace string Kubernetes namespace (overrides cluster name). @@ -565,7 +560,6 @@ It has following subcommands: --batch-ids strings Comma separated list of postage batch IDs to set. If not provided, all batches are set. --cluster-name string Target Beekeeper cluster name. --dilution-depth uint16 Number of levels by which to increase the depth of a stamp during dilution. (default 1) - --geth-url string Geth URL for chain state retrieval. --help help for set --label-selector string Kubernetes label selector for filtering resources (use empty string for all). (default "beekeeper.ethswarm.org/node-funder=true") --namespace string Kubernetes namespace (overrides cluster name). @@ -603,6 +597,7 @@ example: --config-git-repo string URL of the Git repository containing configuration files (uses the config-dir if not specified) --config-git-username string Git username for authentication (required for private repositories) --enable-k8s Enable Kubernetes client functionality (default true) +--geth-url string URL of the Ethereum-compatible blockchain RPC endpoint --in-cluster Use the in-cluster Kubernetes client --kubeconfig string Path to the kubeconfig file (default "~/.kube/config") --log-verbosity string Log verbosity level (0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace) (default "info") diff --git a/cmd/beekeeper/cmd/check.go b/cmd/beekeeper/cmd/check.go index f1f69ae61..eb1b0bade 100644 --- a/cmd/beekeeper/cmd/check.go +++ b/cmd/beekeeper/cmd/check.go @@ -14,8 +14,6 @@ import ( "github.com/spf13/cobra" ) -var errMissingClusterName = fmt.Errorf("cluster name not provided") - func (c *command) initCheckCmd() error { const ( optionNameCreateCluster = "create-cluster" @@ -89,7 +87,8 @@ func (c *command) initCheckCmd() error { // set global config checkGlobalConfig := config.CheckGlobalConfig{ - Seed: c.globalConfig.GetInt64(optionNameSeed), + Seed: c.globalConfig.GetInt64(optionNameSeed), + GethURL: c.globalConfig.GetString(optionNameGethURL), } // run checks @@ -124,6 +123,7 @@ func (c *command) initCheckCmd() error { defer cancelCheck() c.log.Infof("running check: %s", checkName) + c.log.Debugf("check options: %+v", o) ch := make(chan error, 1) go func() { diff --git a/cmd/beekeeper/cmd/cluster.go b/cmd/beekeeper/cmd/cluster.go index 37e4cf6e3..c6f24a08f 100644 --- a/cmd/beekeeper/cmd/cluster.go +++ b/cmd/beekeeper/cmd/cluster.go @@ -132,12 +132,15 @@ func (c *command) setupCluster(ctx context.Context, clusterName string, startClu if clusterConfig.IsUsingStaticEndpoints() { return nil, errors.New("static endpoints are not supported for starting the cluster") } - if chainNodeEndpoint = c.globalConfig.GetString(optionNameChainNodeEndpoint); chainNodeEndpoint == "" { - return nil, errors.New("chain node endpoint (geth-url) not provided") + + if chainNodeEndpoint = c.globalConfig.GetString(optionNameGethURL); chainNodeEndpoint == "" { + return nil, errBlockchainEndpointNotProvided } + if walletKey = c.globalConfig.GetString(optionNameWalletKey); walletKey == "" { return nil, errors.New("wallet key not provided") } + fundOpts = ensureFundingDefaults(clusterConfig.Funding.Export(), c.log) } diff --git a/cmd/beekeeper/cmd/cmd.go b/cmd/beekeeper/cmd/cmd.go index 08c8221ff..501f2dd95 100644 --- a/cmd/beekeeper/cmd/cmd.go +++ b/cmd/beekeeper/cmd/cmd.go @@ -18,7 +18,7 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" - httptransport "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -27,11 +27,15 @@ import ( const ( optionNameConfigDir = "config-dir" - optionNameConfigGitRepo = "config-git-repo" - optionNameConfigGitDir = "config-git-dir" optionNameConfigGitBranch = "config-git-branch" - optionNameConfigGitUsername = "config-git-username" + optionNameConfigGitDir = "config-git-dir" optionNameConfigGitPassword = "config-git-password" + optionNameConfigGitRepo = "config-git-repo" + optionNameConfigGitUsername = "config-git-username" + optionNameEnableK8S = "enable-k8s" + optionNameGethURL = "geth-url" + optionNameInCluster = "in-cluster" + optionNameKubeconfig = "kubeconfig" optionNameLogVerbosity = "log-verbosity" optionNameLokiEndpoint = "loki-endpoint" optionNameTracingEnabled = "tracing-enable" @@ -39,9 +43,11 @@ const ( optionNameTracingHost = "tracing-host" optionNameTracingPort = "tracing-port" optionNameTracingServiceName = "tracing-service-name" - optionNameEnableK8S = "enable-k8s" - optionNameInCluster = "in-cluster" - optionNameKubeconfig = "kubeconfig" +) + +var ( + errBlockchainEndpointNotProvided = errors.New("URL of the Ethereum-compatible blockchain RPC endpoint not provided; use the --geth-url flag") + errMissingClusterName = errors.New("cluster name not provided") ) func init() { @@ -154,6 +160,7 @@ func (c *command) initGlobalFlags() { globalFlags.String(optionNameConfigGitBranch, "main", "Git branch to use for configuration files") globalFlags.String(optionNameConfigGitUsername, "", "Git username for authentication (required for private repositories)") globalFlags.String(optionNameConfigGitPassword, "", "Git password or personal access token for authentication (required for private repositories)") + globalFlags.String(optionNameGethURL, "", "URL of the ethereum compatible blockchain RPC endpoint") globalFlags.String(optionNameLogVerbosity, "info", "Log verbosity level (0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace)") globalFlags.String(optionNameLokiEndpoint, "", "HTTP endpoint for sending logs to Loki (e.g., http://loki.testnet.internal/loki/api/v1/push)") globalFlags.Bool(optionNameTracingEnabled, false, "Enable tracing for performance monitoring and debugging") @@ -169,11 +176,12 @@ func (c *command) initGlobalFlags() { func (c *command) bindGlobalFlags() error { for _, flag := range []string{ optionNameConfigDir, - optionNameConfigGitRepo, optionNameConfigGitBranch, optionNameConfigGitDir, - optionNameConfigGitUsername, optionNameConfigGitPassword, + optionNameConfigGitRepo, + optionNameConfigGitUsername, + optionNameGethURL, optionNameLogVerbosity, optionNameLokiEndpoint, } { @@ -255,7 +263,7 @@ func (c *command) loadConfigDirectory() error { // read configuration from git repo fs := memfs.New() if _, err := git.Clone(memory.NewStorage(), fs, &git.CloneOptions{ - Auth: &httptransport.BasicAuth{ + Auth: &http.BasicAuth{ Username: c.globalConfig.GetString(optionNameConfigGitUsername), Password: c.globalConfig.GetString(optionNameConfigGitPassword), }, @@ -412,8 +420,8 @@ func (c *command) executePeriodically(ctx context.Context, action func(ctx conte } func (c *command) setSwapClient() (err error) { - if len(c.globalConfig.GetString("geth-url")) > 0 { - gethUrl, err := url.Parse(c.globalConfig.GetString("geth-url")) + if c.globalConfig.IsSet(optionNameGethURL) { + gethUrl, err := url.Parse(c.globalConfig.GetString(optionNameGethURL)) if err != nil { return fmt.Errorf("parsing Geth URL: %w", err) } diff --git a/cmd/beekeeper/cmd/create_bee_cluster.go b/cmd/beekeeper/cmd/create_bee_cluster.go index 2661b0649..0a0e109f3 100644 --- a/cmd/beekeeper/cmd/create_bee_cluster.go +++ b/cmd/beekeeper/cmd/create_bee_cluster.go @@ -8,10 +8,9 @@ import ( ) const ( - optionNameClusterName string = "cluster-name" - optionNameChainNodeEndpoint string = "geth-url" - optionNameWalletKey string = "wallet-key" - optionNameTimeout string = "timeout" + optionNameClusterName string = "cluster-name" + optionNameWalletKey string = "wallet-key" + optionNameTimeout string = "timeout" ) func (c *command) initCreateBeeCluster() *cobra.Command { @@ -31,7 +30,6 @@ func (c *command) initCreateBeeCluster() *cobra.Command { } cmd.Flags().String(optionNameClusterName, "", "cluster name") - cmd.Flags().String(optionNameChainNodeEndpoint, "", "Endpoint to chain node. Required.") cmd.Flags().String(optionNameWalletKey, "", "Hex-encoded private key for the Bee node wallet. Required.") cmd.Flags().Duration(optionNameTimeout, 30*time.Minute, "timeout") diff --git a/cmd/beekeeper/cmd/fund.go b/cmd/beekeeper/cmd/fund.go index 21dc1f844..4e430fc43 100644 --- a/cmd/beekeeper/cmd/fund.go +++ b/cmd/beekeeper/cmd/fund.go @@ -17,7 +17,6 @@ func (c *command) initFundCmd() (err error) { optionNameAddressCount = "address-count" optionNameEthAccount = "eth-account" optionNameBzzTokenAddress = "bzz-token-address" - optionNameGethURL = "geth-url" optionNameBzzDeposit = "bzz-deposit" optionNameEthDeposit = "eth-deposit" optionNameGBzzDeposit = "gBzz-deposit" @@ -109,7 +108,6 @@ beekeeper fund --address-create --address-count 2 --bzz-deposit 100.0 --eth-depo cmd.Flags().Int(optionNameAddressCount, 1, "number of Ethereum addresses to create") cmd.Flags().String(optionNameBzzTokenAddress, "0x6aab14fe9cccd64a502d23842d916eb5321c26e7", "BZZ token address") cmd.Flags().String(optionNameEthAccount, "0x62cab2b3b55f341f10348720ca18063cdb779ad5", "ETH account address") - cmd.Flags().String(optionNameGethURL, "http://geth-swap.geth-swap.staging.internal", "Geth node URL") cmd.Flags().Float64(optionNameBzzDeposit, 0, "BZZ tokens amount to deposit") cmd.Flags().Float64(optionNameGBzzDeposit, 0, "gBZZ tokens amount to deposit") cmd.Flags().Float64(optionNameEthDeposit, 0, "ETH amount to deposit") diff --git a/cmd/beekeeper/cmd/node_funder.go b/cmd/beekeeper/cmd/node_funder.go index e18a13928..45cd1f38a 100644 --- a/cmd/beekeeper/cmd/node_funder.go +++ b/cmd/beekeeper/cmd/node_funder.go @@ -18,14 +18,13 @@ const ( func (c *command) initNodeFunderCmd() (err error) { const ( - optionNameAddresses = "addresses" - optionNameNamespace = "namespace" - optionNameChainNodeEndpoint = "geth-url" - optionNameWalletKey = "wallet-key" - optionNameMinNative = "min-native" - optionNameMinSwarm = "min-swarm" - optionNameTimeout = "timeout" - optionNameLabelSelector = "label-selector" + optionNameAddresses = "addresses" + optionNameNamespace = "namespace" + optionNameWalletKey = "wallet-key" + optionNameMinNative = "min-native" + optionNameMinSwarm = "min-swarm" + optionNameTimeout = "timeout" + optionNameLabelSelector = "label-selector" ) cmd := &cobra.Command{ @@ -41,8 +40,8 @@ func (c *command) initNodeFunderCmd() (err error) { }, } - if cfg.ChainNodeEndpoint = c.globalConfig.GetString(optionNameChainNodeEndpoint); cfg.ChainNodeEndpoint == "" { - return errors.New("chain node endpoint (geth-url) not provided") + if cfg.ChainNodeEndpoint = c.globalConfig.GetString(optionNameGethURL); cfg.ChainNodeEndpoint == "" { + return errBlockchainEndpointNotProvided } if cfg.WalletKey = c.globalConfig.GetString(optionNameWalletKey); cfg.WalletKey == "" { @@ -51,13 +50,13 @@ func (c *command) initNodeFunderCmd() (err error) { defer c.log.Infof("node-funder done") - logger := funder.WithLoggerOption(c.log) + logOpt := funder.WithLoggerOption(c.log) addresses := c.globalConfig.GetStringSlice(optionNameAddresses) if len(addresses) > 0 { cfg.Addresses = addresses return c.executePeriodically(ctx, func(ctx context.Context) error { - return funder.Fund(ctx, cfg, nil, nil, logger) + return funder.Fund(ctx, cfg, nil, nil, logOpt) }) } @@ -68,7 +67,7 @@ func (c *command) initNodeFunderCmd() (err error) { cfg.Namespace = namespace return c.executePeriodically(ctx, func(ctx context.Context) error { - return funder.Fund(ctx, cfg, funderClient, nil, logger) + return funder.Fund(ctx, cfg, funderClient, nil, logOpt) }) } @@ -93,7 +92,7 @@ func (c *command) initNodeFunderCmd() (err error) { } return c.executePeriodically(ctx, func(ctx context.Context) error { - return funder.Fund(ctx, cfg, nil, nil, logger) + return funder.Fund(ctx, cfg, nil, nil, logOpt) }) } @@ -111,7 +110,6 @@ func (c *command) initNodeFunderCmd() (err error) { cmd.Flags().StringSliceP(optionNameAddresses, "a", nil, "Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name.") cmd.Flags().StringP(optionNameNamespace, "n", "", "Kubernetes namespace. Overrides cluster name if set.") cmd.Flags().String(optionNameClusterName, "", "Name of the Beekeeper cluster to target. Ignored if a namespace is specified.") - cmd.Flags().String(optionNameChainNodeEndpoint, "", "Endpoint to chain node. Required.") cmd.Flags().String(optionNameWalletKey, "", "Hex-encoded private key for the Bee node wallet. Required.") cmd.Flags().Float64(optionNameMinNative, 0, "Minimum amount of chain native coins (xDAI) nodes should have.") cmd.Flags().Float64(optionNameMinSwarm, 0, "Minimum amount of swarm tokens (xBZZ) nodes should have.") diff --git a/cmd/beekeeper/cmd/operator.go b/cmd/beekeeper/cmd/operator.go index bf1bcd689..2130593cd 100644 --- a/cmd/beekeeper/cmd/operator.go +++ b/cmd/beekeeper/cmd/operator.go @@ -11,13 +11,12 @@ import ( func (c *command) initOperatorCmd() error { const ( - optionNameNamespace = "namespace" - optionNameChainNodeEndpoint = "geth-url" - optionNameWalletKey = "wallet-key" - optionNameMinNative = "min-native" - optionNameMinSwarm = "min-swarm" - optionNameTimeout = "timeout" - optionNameLabelSelector = "label-selector" + optionNameNamespace = "namespace" + optionNameWalletKey = "wallet-key" + optionNameMinNative = "min-native" + optionNameMinSwarm = "min-swarm" + optionNameTimeout = "timeout" + optionNameLabelSelector = "label-selector" ) cmd := &cobra.Command{ @@ -31,9 +30,8 @@ func (c *command) initOperatorCmd() error { return errors.New("namespace not provided") } - chainNodeEndpoint := c.globalConfig.GetString(optionNameChainNodeEndpoint) - if chainNodeEndpoint == "" { - return errors.New("chain node endpoint (geth-url) not provided") + if !c.globalConfig.IsSet(optionNameGethURL) { + return errBlockchainEndpointNotProvided } walletKey := c.globalConfig.GetString(optionNameWalletKey) @@ -45,7 +43,7 @@ func (c *command) initOperatorCmd() error { Log: c.log, Namespace: namespace, WalletKey: walletKey, - ChainNodeEndpoint: chainNodeEndpoint, + ChainNodeEndpoint: c.globalConfig.GetString(optionNameGethURL), NativeToken: c.globalConfig.GetFloat64(optionNameMinNative), SwarmToken: c.globalConfig.GetFloat64(optionNameMinSwarm), K8sClient: c.k8sClient, @@ -57,7 +55,6 @@ func (c *command) initOperatorCmd() error { } cmd.Flags().StringP(optionNameNamespace, "n", "", "Kubernetes namespace to scan for scheduled pods.") - cmd.Flags().String(optionNameChainNodeEndpoint, "", "Endpoint to chain node. Required.") cmd.Flags().String(optionNameWalletKey, "", "Hex-encoded private key for the Bee node wallet. Required.") cmd.Flags().Float64(optionNameMinNative, 0, "Minimum amount of chain native coins (xDAI) nodes should have.") cmd.Flags().Float64(optionNameMinSwarm, 0, "Minimum amount of swarm tokens (xBZZ) nodes should have.") diff --git a/cmd/beekeeper/cmd/stamper.go b/cmd/beekeeper/cmd/stamper.go index 5d068c8f1..0701b8c44 100644 --- a/cmd/beekeeper/cmd/stamper.go +++ b/cmd/beekeeper/cmd/stamper.go @@ -47,10 +47,10 @@ func initStamperDefaultFlags(cmd *cobra.Command) *cobra.Command { func (c *command) initStamperTopup() *cobra.Command { const ( - optionNameTTLThreshold = "ttl-threshold" - optionNameTopUpTo = "topup-to" - optionNameGethUrl = "geth-url" - optionNameBatchIDs = "batch-ids" + optionNameTTLThreshold = "ttl-threshold" + optionNameTopUpTo = "topup-to" + optionNameBatchIDs = "batch-ids" + optionNamePostageLabels = "postage-labels" ) cmd := &cobra.Command{ @@ -69,6 +69,7 @@ func (c *command) initStamperTopup() *cobra.Command { c.globalConfig.GetDuration(optionNameTTLThreshold), c.globalConfig.GetDuration(optionNameTopUpTo), stamper.WithBatchIDs(c.globalConfig.GetStringSlice(optionNameBatchIDs)), + stamper.WithPostageLabels(c.globalConfig.GetStringSlice(optionNamePostageLabels)), ) }) }) @@ -78,8 +79,8 @@ func (c *command) initStamperTopup() *cobra.Command { cmd.Flags().Duration(optionNameTTLThreshold, 5*24*time.Hour, "Threshold for the remaining TTL of a stamp. Actions are triggered when TTL drops below this value.") cmd.Flags().Duration(optionNameTopUpTo, 30*24*time.Hour, "Duration to top up the TTL of a stamp to.") - cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to top up. If not provided, all batches are topped up.") - cmd.Flags().String(optionNameGethUrl, "", "Geth URL for chain state retrieval.") + cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to top up. If not provided, all batches are topped up. Overides postage labels.") + cmd.Flags().StringSlice(optionNamePostageLabels, nil, "Comma separated list of postage labels to top up. If not provided, all batches are topped up.") cmd.Flags().Duration(optionNamePeriodicCheck, 0, "Periodic check interval. Default is 0, which means no periodic check.") return cmd @@ -90,6 +91,7 @@ func (c *command) initStamperDilute() *cobra.Command { optionNameUsageThreshold = "usage-threshold" optionNameDiutionDepth = "dilution-depth" optionNameBatchIDs = "batch-ids" + optionNamePostageLabels = "postage-labels" ) cmd := &cobra.Command{ @@ -108,6 +110,7 @@ func (c *command) initStamperDilute() *cobra.Command { c.globalConfig.GetFloat64(optionNameUsageThreshold), c.globalConfig.GetUint16(optionNameDiutionDepth), stamper.WithBatchIDs(c.globalConfig.GetStringSlice(optionNameBatchIDs)), + stamper.WithPostageLabels(c.globalConfig.GetStringSlice(optionNamePostageLabels)), ) }) }) @@ -117,7 +120,8 @@ func (c *command) initStamperDilute() *cobra.Command { cmd.Flags().Float64(optionNameUsageThreshold, 90, "Percentage threshold for stamp utilization. Triggers dilution when usage exceeds this value.") cmd.Flags().Uint8(optionNameDiutionDepth, 1, "Number of levels by which to increase the depth of a stamp during dilution.") - cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to dilute. If not provided, all batches are diluted.") + cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to dilute. If not provided, all batches are diluted. Overides postage labels.") + cmd.Flags().StringSlice(optionNamePostageLabels, nil, "Comma separated list of postage labels to top up. If not provided, all batches are topped up.") cmd.Flags().Duration(optionNamePeriodicCheck, 0, "Periodic check interval. Default is 0, which means no periodic check.") return cmd @@ -125,8 +129,9 @@ func (c *command) initStamperDilute() *cobra.Command { func (c *command) initStamperCreate() *cobra.Command { const ( - optionNameAmount = "amount" - optionNameDepth = "depth" + optionNameDuration = "duration" + optionNameDepth = "depth" + optionNamePostageLabel = "postage-label" ) cmd := &cobra.Command{ @@ -141,16 +146,18 @@ func (c *command) initStamperCreate() *cobra.Command { } return stamperClient.Create(ctx, - c.globalConfig.GetUint64(optionNameAmount), + c.globalConfig.GetDuration(optionNameDuration), c.globalConfig.GetUint16(optionNameDepth), + c.globalConfig.GetString(optionNamePostageLabel), ) }) }, PreRunE: c.preRunE, } - cmd.Flags().Uint64(optionNameAmount, 100000000, "Amount of BZZ in PLURS added that the postage batch will have.") + cmd.Flags().Duration(optionNameDuration, 24*time.Hour, "Duration of the postage batch") cmd.Flags().Uint16(optionNameDepth, 17, "Batch depth which specifies how many chunks can be signed with the batch. It is a logarithm. Must be higher than default bucket depth (16)") + cmd.Flags().String(optionNamePostageLabel, "beekeeper", "Postage label for the batch") return cmd } @@ -161,8 +168,8 @@ func (c *command) initStamperSet() *cobra.Command { optionNameTopUpTo = "topup-to" optionNameUsageThreshold = "usage-threshold" optionNameDiutionDepth = "dilution-depth" - optionNameGethUrl = "geth-url" optionNameBatchIDs = "batch-ids" + optionNamePostageLabels = "postage-labels" ) cmd := &cobra.Command{ @@ -183,6 +190,7 @@ func (c *command) initStamperSet() *cobra.Command { c.globalConfig.GetFloat64(optionNameUsageThreshold), c.globalConfig.GetUint16(optionNameDiutionDepth), stamper.WithBatchIDs(c.globalConfig.GetStringSlice(optionNameBatchIDs)), + stamper.WithPostageLabels(c.globalConfig.GetStringSlice(optionNamePostageLabels)), ) }) }) @@ -194,8 +202,8 @@ func (c *command) initStamperSet() *cobra.Command { cmd.Flags().Duration(optionNameTopUpTo, 30*24*time.Hour, "Duration to top up the TTL of a stamp to.") cmd.Flags().Float64(optionNameUsageThreshold, 90, "Percentage threshold for stamp utilization. Triggers dilution when usage exceeds this value.") cmd.Flags().Uint16(optionNameDiutionDepth, 1, "Number of levels by which to increase the depth of a stamp during dilution.") - cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to set. If not provided, all batches are set.") - cmd.Flags().String(optionNameGethUrl, "", "Geth URL for chain state retrieval.") + cmd.Flags().StringSlice(optionNameBatchIDs, nil, "Comma separated list of postage batch IDs to set. If not provided, all batches are set. Overides postage labels.") + cmd.Flags().StringSlice(optionNamePostageLabels, nil, "Comma separated list of postage labels to set. If not provided, all batches are set.") cmd.Flags().Duration(optionNamePeriodicCheck, 0, "Periodic check interval. Default is 0, which means no periodic check.") return cmd diff --git a/config/config.yaml b/config/config.yaml index 906bdddb1..4e98b1e3d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -141,7 +141,7 @@ checks: options: file-size: 1024 postage-depth: 20 - postage-amount: 420000000 + postage-ttl: 72h postage-label: act-label seed: 0 timeout: 5m @@ -151,7 +151,7 @@ checks: dry-run: false file-name: balances file-size: 1048576 # 1mb = 1*1024*1024 - postage-amount: 1000 + postage-ttl: 24h postage-depth: 16 upload-node-count: 3 wait-before-download: 5s @@ -174,7 +174,8 @@ checks: file-size: 1048576 # 1mb = 1*1024*1024 files-per-node: 1 full: false - postage-amount: 1000 + postage-ttl: 24h + postage-label: test-label upload-node-count: 3 timeout: 5m type: file-retrieval @@ -210,15 +211,17 @@ checks: options: files-in-collection: 10 max-pathname-length: 64 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label timeout: 30m type: manifest networkavailability: options: rnd-seed: - postage-amount: 50000000 - postage-depth: 24 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label sleep-duration: 1h timeout: 5m type: networkavailability @@ -241,15 +244,16 @@ checks: options: count: 3 address-prefix: 2 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 5m type: pss pullsync: options: chunks-per-node: 1 - postage-amount: 1000 + postage-ttl: 24h replication-factor-threshold: 2 upload-node-count: 1 timeout: 5m @@ -258,8 +262,9 @@ checks: options: chunks-per-node: 1 mode: default - postage-amount: 1000 - postage-depth: 16 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 1s upload-node-count: 1 @@ -269,8 +274,9 @@ checks: options: chunks-per-node: 3 mode: chunks - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 15s upload-node-count: 3 @@ -283,8 +289,9 @@ checks: options: chunks-per-node: 3 mode: light-chunks - postage-amount: 10000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 30s upload-node-count: 3 @@ -295,16 +302,18 @@ checks: type: pushsync redundancy: options: - postage-amount: - postage-depth: + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label seed: data-size: type: redundancy retrieval: options: chunks-per-node: 3 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label upload-node-count: 3 timeout: 5m type: retrieval @@ -314,8 +323,9 @@ checks: expect-settlements: true file-name: settlements file-size: 14680064 # 14mb = 14*1024*1024 - postage-amount: 1000 - postage-depth: 20 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label threshold: 100000000 upload-node-count: 3 wait-before-download: 15s @@ -324,8 +334,9 @@ checks: smoke: options: content-size: 5000 - postage-amount: 1000000 - postage-depth: 20 + postage-ttl: 72h + postage-depth: 21 + postage-label: test-label nodes-sync-wait: 1m upload-timeout: 1m download-timeout: 1m @@ -335,8 +346,9 @@ checks: type: smoke soc: options: - postage-amount: 1000 - postage-depth: 16 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 5m type: soc @@ -348,26 +360,18 @@ checks: insufficient-amount: 102400 contract-addr: "0xfc28330f1ecE0ef2371B724E0D19c1EE60B728b2" private-key: "4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" - geth-url: "http://geth-swap.bee-playground.svc.swarm1.local:8545" - geth-chain-id: 12345 withdraw: options: target-address: 0xec44cb15b1b033e74d55ac5d0e24d861bde54532 timeout: 5m type: withdraw - content-availability: - type: content-availability - timeout: 5m - options: - content-size: 16384 - postage-amount: 1000 - postage-depth: 16 load: options: content-size: 500000 #5000000 - postage-amount: 1000000 gas-price: 10000000000 - postage-depth: 20 + postage-ttl: 72h + postage-depth: 21 + postage-label: test-label nodes-sync-wait: 2m duration: 12h downloader-count: 3 @@ -380,20 +384,18 @@ checks: - light timeout: 12h type: load - gsoc: options: - postage-amount: 100000 - postage-depth: 20 - postage-label: gsoc-label + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label timeout: 10m type: gsoc - feed: options: - postage-amount: 100000 - postage-depth: 20 - postage-label: feed-label + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label type: feed # simulations defines simulations Beekeeper can execute against the cluster @@ -405,7 +407,7 @@ simulations: gas-price: "10000000000" max-file-size: 2097152 # 2mb = 2*1024*1024 min-file-size: 1048576 # 1mb = 1*1024*1024 - postage-amount: 1000 + postage-ttl: 24h postage-depth: 16 retries: 5 retry-delay: 1s @@ -419,7 +421,7 @@ simulations: options: chunks-per-node: 1 gas-price: "10000000000" - postage-amount: 1000 + postage-ttl: 24h postage-depth: 16 upload-node-count: 1 upload-delay: 10s @@ -427,7 +429,7 @@ simulations: type: retrieval pushsync: options: - postage-amount: 1000 + postage-ttl: 24h postage-depth: 20 seed: proxy-api-endpoint: "http://ethproxy.localhost" diff --git a/config/local.yaml b/config/local.yaml index 2d4b065c2..84aa8f5a8 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -175,9 +175,9 @@ checks: ci-act: options: file-size: 1024 - postage-depth: 20 - postage-amount: 420000000 - postage-label: act-label + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label seed: 0 timeout: 5m type: act @@ -202,8 +202,9 @@ checks: options: files-in-collection: 10 max-pathname-length: 64 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label timeout: 30m type: manifest ci-pingpong: @@ -219,8 +220,9 @@ checks: options: count: 3 address-prefix: 2 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 5m type: pss @@ -228,8 +230,9 @@ checks: options: chunks-per-node: 3 mode: chunks - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 15s upload-node-count: 3 @@ -242,8 +245,9 @@ checks: options: chunks-per-node: 3 mode: light-chunks - postage-amount: 10000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 30s upload-node-count: 3 @@ -255,8 +259,9 @@ checks: ci-retrieval: options: chunks-per-node: 3 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label upload-node-count: 3 timeout: 5m type: retrieval @@ -266,8 +271,9 @@ checks: expect-settlements: true file-name: settlements file-size: 14680064 # 14mb = 14*1024*1024 - postage-amount: 1000 - postage-depth: 20 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label threshold: 100000000 upload-node-count: 3 wait-before-download: 15s @@ -276,8 +282,9 @@ checks: ci-smoke: options: content-size: 5000000 - postage-amount: 1000000 - postage-depth: 20 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label nodes-sync-wait: 1m upload-timeout: 1m download-timeout: 1m @@ -286,8 +293,9 @@ checks: type: smoke ci-soc: options: - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 5m type: soc @@ -296,8 +304,9 @@ checks: timeout: 5m options: content-size: 16384 - postage-amount: 1000 - postage-depth: 17 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label ci-postage: type: postage timeout: 5m @@ -314,8 +323,6 @@ checks: insufficient-amount: 102400 contract-addr: "0xfc28330f1ecE0ef2371B724E0D19c1EE60B728b2" private-key: "4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" - geth-url: "http://geth-swap.localhost" - geth-chain-id: 12345 ci-longavailability: options: refs: @@ -330,22 +337,22 @@ checks: type: datadurability ci-redundancy: options: - postage-amount: - postage-depth: + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label seed: data-size: type: redundancy ci-gsoc: options: - postage-amount: 100000 - postage-depth: 20 - postage-label: gsoc-label + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label timeout: 10m type: gsoc ci-feed: options: - postage-amount: 100000 - postage-depth: 20 - postage-label: feed-label + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label type: feed - diff --git a/config/public-testnet.yaml b/config/public-testnet.yaml index 04bb4fdfc..7593ad198 100644 --- a/config/public-testnet.yaml +++ b/config/public-testnet.yaml @@ -57,8 +57,9 @@ checks: pt-retrieval: options: chunks-per-node: 3 - postage-amount: 140000000 - postage-depth: 17 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label upload-node-count: 3 timeout: 15m type: retrieval @@ -68,8 +69,9 @@ checks: expect-settlements: true file-name: settlements file-size: 14680064 # 14mb = 14*1024*1024 - postage-amount: 140000000 - postage-depth: 20 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label threshold: 100000000 upload-node-count: 3 wait-before-download: 15s @@ -79,23 +81,26 @@ checks: options: files-in-collection: 10 max-pathname-length: 64 - postage-amount: 140000000 - postage-depth: 17 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label timeout: 30m type: manifest pt-pss: options: count: 3 address-prefix: 2 - postage-amount: 140000000 - postage-depth: 17 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 30m type: pss pt-soc: options: - postage-amount: 140000000 - postage-depth: 17 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label request-timeout: 5m timeout: 5m type: soc @@ -103,8 +108,9 @@ checks: options: chunks-per-node: 3 mode: chunks - postage-amount: 140000000 - postage-depth: 17 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label retries: 5 retry-delay: 15s upload-node-count: 3 @@ -123,16 +129,16 @@ checks: postage-new-depth: 18 pt-gsoc: options: - postage-amount: 140000000 - postage-depth: 20 - postage-label: gsoc-label + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label timeout: 10m type: gsoc pt-testnet-load: options: content-size: 50000000 - postage-amount: 6120360000 # (28335/12)*60*60*24*30 - postage-depth: 24 + postage-ttl: 720h + postage-depth: 21 postage-label: test-label duration: 720h uploader-count: 2 @@ -149,9 +155,9 @@ checks: type: load pt-feed: options: - postage-amount: 140000000 - postage-depth: 20 - postage-label: feed-label + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label timeout: 30m type: feed pt-feed-availability: diff --git a/config/staging.yaml b/config/staging.yaml index cd5b55b15..3ac5f1d0c 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -51,8 +51,9 @@ checks: st-testnet-load: options: content-size: 50000000 - postage-amount: 1000000000 - postage-depth: 24 + postage-ttl: 336h + postage-depth: 21 + postage-label: test-label duration: 12h uploader-count: 2 downloader-count: 0 diff --git a/config/testnet-giant.yaml b/config/testnet-giant.yaml index 9199e7273..1984ac13b 100644 --- a/config/testnet-giant.yaml +++ b/config/testnet-giant.yaml @@ -107,9 +107,3 @@ bee-configs: bootnode-giant: _inherit: "testnet-giant" bootnode-mode: true - -checks: - testnet-giant-pingpong: - options: - timeout: 5m - type: pingpong diff --git a/pkg/bee/api/postage.go b/pkg/bee/api/postage.go index eb3f100e4..75d00b14b 100644 --- a/pkg/bee/api/postage.go +++ b/pkg/bee/api/postage.go @@ -42,11 +42,11 @@ func (p *PostageService) CreatePostageBatch(ctx context.Context, amount int64, d h := http.Header{} h.Add("Immutable", "false") - err = p.client.requestWithHeader(ctx, http.MethodPost, url, h, nil, &resp) - if err != nil { + if err := p.client.requestWithHeader(ctx, http.MethodPost, url, h, nil, &resp); err != nil { return "", err } - return resp.BatchID, err + + return resp.BatchID, nil } // Sends a topup batch request to a node that returns the batchID @@ -74,17 +74,18 @@ func (p *PostageService) DilutePostageBatch(ctx context.Context, batchID string, // Fetches the list postage stamp batches func (p *PostageService) PostageBatches(ctx context.Context) ([]PostageStampResponse, error) { var resp postageStampsResponse - err := p.client.request(ctx, http.MethodGet, "/stamps", nil, &resp) - if err != nil { + + if err := p.client.request(ctx, http.MethodGet, "/stamps", nil, &resp); err != nil { return nil, err } + return resp.Stamps, nil } func (p *PostageService) PostageStamp(ctx context.Context, batchID string) (PostageStampResponse, error) { var resp PostageStampResponse - err := p.client.request(ctx, http.MethodGet, "/stamps/"+batchID, nil, &resp) - if err != nil { + + if err := p.client.request(ctx, http.MethodGet, "/stamps/"+batchID, nil, &resp); err != nil { return PostageStampResponse{}, err } return resp, nil @@ -116,13 +117,15 @@ type ChainStateResponse struct { // GetChainState returns the chain state of the node func (p *PostageService) GetChainState(ctx context.Context) (ChainStateResponse, error) { var resp ChainStateResponse - err := p.client.request(ctx, http.MethodGet, "/chainstate", nil, &resp) - if err != nil { + + if err := p.client.request(ctx, http.MethodGet, "/chainstate", nil, &resp); err != nil { return ChainStateResponse{}, err } + return resp, nil } +// BatchUsage returns the batch usage in percent func (batch *PostageStampResponse) BatchUsage() float64 { maxUtilization := 1 << (batch.Depth - batch.BucketDepth) // 2^(depth - bucketDepth) return (float64(batch.Utilization) / float64(maxUtilization)) * 100 // batch utilization between 0 and 100 percent diff --git a/pkg/bee/client.go b/pkg/bee/client.go index ee84cd492..9b69e84a4 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -19,15 +19,18 @@ import ( "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/beekeeper/pkg/bee/api" "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/swap" ) const retryCount int = 5 // Client manages communication with the Bee node type Client struct { - api *api.Client - opts ClientOptions - log logging.Logger + api *api.Client + swapClient swap.Client + log logging.Logger + name string + apiURL *url.URL // number of times to retry call retry int } @@ -38,14 +41,17 @@ type ClientOptions struct { APIURL *url.URL Name string Retry int + SwapClient swap.Client } // NewClient returns Bee client func NewClient(opts ClientOptions, log logging.Logger) (c *Client) { c = &Client{ - retry: retryCount, - opts: opts, - log: log, + retry: retryCount, + log: log, + swapClient: opts.SwapClient, + name: opts.Name, + apiURL: opts.APIURL, } httpClient := &http.Client{ @@ -76,11 +82,11 @@ type Addresses struct { } func (c *Client) Name() string { - return c.opts.Name + return c.name } -func (c *Client) Config() ClientOptions { - return c.opts +func (c *Client) Host() string { + return c.apiURL.Host } func (c *Client) API() *api.Client { @@ -448,10 +454,29 @@ func (c *Client) CreatePostageBatch(ctx context.Context, amount int64, depth uin return id, nil } -func (c *Client) GetOrCreateMutableBatch(ctx context.Context, amount int64, depth uint64, label string) (string, error) { +func (c *Client) GetOrCreateMutableBatch(ctx context.Context, postageTTL time.Duration, depth uint64, label string) (string, error) { + csr, err := c.api.Postage.GetChainState(ctx) + if err != nil { + return "", fmt.Errorf("get chain state: %w", err) + } + + blockTime, err := c.swapClient.FetchBlockTime(ctx, swap.WithOffset(1000)) + if err != nil { + return "", fmt.Errorf("fetching block time: %w", err) + } + + amount := int64(1000) + + price := csr.CurrentPrice.Int64() + if price > 0 { + amount = (int64(postageTTL.Seconds()) / blockTime) * price + } else { + c.log.Warningf("invalid chain price: %d", price) + } + batches, err := c.PostageBatches(ctx) if err != nil { - return "", err + return "", fmt.Errorf("get postage batches: %w", err) } for _, b := range batches { diff --git a/pkg/check/act/act.go b/pkg/check/act/act.go index a81428c81..edab65820 100644 --- a/pkg/check/act/act.go +++ b/pkg/check/act/act.go @@ -18,23 +18,23 @@ import ( // Options represents check options type Options struct { - FileName string - FileSize int64 - PostageAmount int64 - PostageDepth uint64 - PostageLabel string - Seed int64 + FileName string + FileSize int64 + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string + Seed int64 } -// NewDefaultOptions returns new default options -func NewDefaultOptions() Options { +// NewOptions returns new default options +func NewOptions() Options { return Options{ - FileName: "act", - FileSize: 1 * 1024, - PostageAmount: 420000000, - PostageDepth: 20, - PostageLabel: "act-label", - Seed: 0, + FileName: "act", + FileSize: 1 * 1024, + PostageTTL: 24 * time.Hour, + PostageDepth: 20, + PostageLabel: "test-label", + Seed: 0, } } @@ -95,7 +95,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int file := bee.NewRandomFile(rnds[0], fileName, o.FileSize) - batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, postagelabel) + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, postagelabel) if err != nil { return fmt.Errorf("created batched id %w", err) } diff --git a/pkg/check/balances/balances.go b/pkg/check/balances/balances.go index 206ee9427..4f88ac2a0 100644 --- a/pkg/check/balances/balances.go +++ b/pkg/check/balances/balances.go @@ -17,7 +17,7 @@ type Options struct { FileName string FileSize int64 GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Seed int64 @@ -32,7 +32,7 @@ func NewDefaultOptions() Options { FileName: "balances", FileSize: 1 * 1024 * 1024, // 1mb, GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", Seed: 0, @@ -70,13 +70,12 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int var checkCase *test.CheckCase caseOpts := test.CaseOptions{ - FileName: o.FileName, - FileSize: o.FileSize, - GasPrice: o.GasPrice, - PostageAmount: o.PostageAmount, - PostageDepth: o.PostageDepth, - PostageLabel: o.PostageLabel, - Seed: o.Seed, + FileName: o.FileName, + FileSize: o.FileSize, + GasPrice: o.GasPrice, + PostageDepth: o.PostageDepth, + PostageLabel: o.PostageLabel, + Seed: o.Seed, } if checkCase, err = test.NewCheckCase(ctx, cluster, caseOpts, c.logger); err != nil { diff --git a/pkg/check/feed/feed.go b/pkg/check/feed/feed.go index ef2d9735a..ec3f33898 100644 --- a/pkg/check/feed/feed.go +++ b/pkg/check/feed/feed.go @@ -18,20 +18,20 @@ import ( // Options represents check options type Options struct { - PostageAmount int64 - PostageDepth uint64 - PostageLabel string - NUpdates int - RootRef string + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string + NUpdates int + RootRef string } // NewDefaultOptions returns new default options func NewDefaultOptions() Options { return Options{ - PostageAmount: 1000, - PostageDepth: 17, - PostageLabel: "test-label", - NUpdates: 2, + PostageTTL: 24 * time.Hour, + PostageDepth: 17, + PostageLabel: "test-label", + NUpdates: 2, } } @@ -104,7 +104,7 @@ func (c *Check) feedCheck(ctx context.Context, cluster orchestration.Cluster, o c.logger.Infof("upload client: %s", upClient.Name()) - batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return err } diff --git a/pkg/check/fileretrieval/fileretrieval.go b/pkg/check/fileretrieval/fileretrieval.go index d878fb269..2236b8e02 100644 --- a/pkg/check/fileretrieval/fileretrieval.go +++ b/pkg/check/fileretrieval/fileretrieval.go @@ -22,7 +22,7 @@ type Options struct { FilesPerNode int Full bool GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageLabel string Seed int64 UploadNodeCount int @@ -36,7 +36,7 @@ func NewDefaultOptions() Options { FilesPerNode: 1, Full: false, GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageLabel: "test-label", Seed: 0, UploadNodeCount: 1, @@ -98,7 +98,7 @@ func (c *Check) defaultCheck(ctx context.Context, cluster orchestration.Cluster, file := bee.NewRandomFile(rnds[i], fmt.Sprintf("%s-%d-%d", o.FileName, i, j), o.FileSize) depth := 2 + bee.EstimatePostageBatchDepth(file.Size()) - batchID, err := clients[nodeName].CreatePostageBatch(ctx, o.PostageAmount, depth, o.PostageLabel, false) + batchID, err := clients[nodeName].GetOrCreateMutableBatch(ctx, o.PostageTTL, depth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: created batched id %w", nodeName, err) } @@ -171,7 +171,7 @@ func (c *Check) fullCheck(ctx context.Context, cluster orchestration.Cluster, o file := bee.NewRandomFile(rnds[i], fmt.Sprintf("%s-%d-%d", o.FileName, i, j), o.FileSize) depth := 2 + bee.EstimatePostageBatchDepth(file.Size()) - batchID, err := clients[nodeName].CreatePostageBatch(ctx, o.PostageAmount, depth, o.PostageLabel, false) + batchID, err := clients[nodeName].GetOrCreateMutableBatch(ctx, o.PostageTTL, depth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: created batched id %w", nodeName, err) } diff --git a/pkg/check/gsoc/gsoc.go b/pkg/check/gsoc/gsoc.go index 9fb7ace6e..af3297db2 100644 --- a/pkg/check/gsoc/gsoc.go +++ b/pkg/check/gsoc/gsoc.go @@ -25,17 +25,17 @@ import ( // Options represents check options type Options struct { - PostageAmount int64 - PostageDepth uint64 - PostageLabel string + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string } // NewDefaultOptions returns new default options func NewDefaultOptions() Options { return Options{ - PostageAmount: 1000, - PostageDepth: 17, - PostageLabel: "test-label", + PostageTTL: 24 * time.Hour, + PostageDepth: 17, + PostageLabel: "test-label", } } @@ -81,8 +81,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int batches := make([]string, 2) for i := 0; i < 2; i++ { - c.logger.Infof("gsoc: creating postage batch. amount=%d, depth=%d, label=%s", o.PostageAmount, o.PostageDepth, o.PostageLabel) - batchID, err := uploadClient.CreatePostageBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel, false) + c.logger.Infof("gsoc: creating postage batch. duration=%d, depth=%d, label=%s", o.PostageTTL, o.PostageDepth, o.PostageLabel) + batchID, err := uploadClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return err } @@ -125,7 +125,7 @@ func run(ctx context.Context, uploadClient *bee.Client, listenClient *bee.Client logger.Infof("gsoc: socAddress=%s, listner node address=%s", socAddress, addresses.Overlay) listener := &socListener{} - ch, err := listener.Listen(ctx, listenClient.Config().APIURL.Host, socAddress, logger) + ch, err := listener.Listen(ctx, listenClient.Host(), socAddress, logger) if err != nil { return fmt.Errorf("listen websocket: %w", err) } diff --git a/pkg/check/manifest/manifest.go b/pkg/check/manifest/manifest.go index 28e59a022..bb787f750 100644 --- a/pkg/check/manifest/manifest.go +++ b/pkg/check/manifest/manifest.go @@ -25,7 +25,7 @@ type Options struct { FilesInCollection int GasPrice string MaxPathnameLength int32 - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Seed int64 @@ -37,7 +37,7 @@ func NewDefaultOptions() Options { FilesInCollection: 10, GasPrice: "", MaxPathnameLength: 64, - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", Seed: 0, @@ -102,7 +102,7 @@ func (c *Check) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Optio } tarFile := bee.NewBufferFile("", tarReader) - batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", upClient.Name(), err) } @@ -132,7 +132,7 @@ func (c *Check) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, return err } - batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", upClient.Name(), err) } diff --git a/pkg/check/networkavailability/check.go b/pkg/check/networkavailability/check.go index 8b4160b71..9679ae0fe 100644 --- a/pkg/check/networkavailability/check.go +++ b/pkg/check/networkavailability/check.go @@ -21,8 +21,9 @@ import ( // Options represents smoke test options type Options struct { RndSeed int64 - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 + PostageLabel string SleepDuration time.Duration } @@ -30,8 +31,9 @@ type Options struct { func NewDefaultOptions() Options { return Options{ RndSeed: time.Now().UnixNano(), - PostageAmount: 50_000_000, + PostageTTL: 24 * time.Hour, PostageDepth: 24, + PostageLabel: "test-label", SleepDuration: time.Hour, } } @@ -102,7 +104,7 @@ iteration: // upload var chunks []swarm.Chunk for _, n := range neighborhoods(int(storageRadius)) { - batch, err := uploadClient.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, "net-avail-check") + batch, err := uploadClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { c.logger.Errorf("create batch failed failed") continue iteration diff --git a/pkg/check/pss/pss.go b/pkg/check/pss/pss.go index 2a727e5f4..e72778474 100644 --- a/pkg/check/pss/pss.go +++ b/pkg/check/pss/pss.go @@ -21,7 +21,7 @@ type Options struct { Count int64 AddressPrefix int GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string RequestTimeout time.Duration @@ -34,7 +34,7 @@ func NewDefaultOptions() Options { Count: 1, AddressPrefix: 1, GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", RequestTimeout: 5 * time.Minute, @@ -122,14 +122,14 @@ func (c *Check) testPss(nodeAName, nodeBName string, clients map[string]*bee.Cli return err } - batchID, err := nodeA.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := nodeA.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { cancel() return fmt.Errorf("node %s: batched id %w", nodeAName, err) } c.logger.Infof("node %s: batched id %s", nodeAName, batchID) - ch, close, err := listenWebsocket(ctx, nodeB.Config().APIURL.Host, testTopic, c.logger) + ch, close, err := listenWebsocket(ctx, nodeB.Host(), testTopic, c.logger) if err != nil { cancel() return err diff --git a/pkg/check/pullsync/pullsync.go b/pkg/check/pullsync/pullsync.go index 3cd5980fe..f1eb5322a 100644 --- a/pkg/check/pullsync/pullsync.go +++ b/pkg/check/pullsync/pullsync.go @@ -20,7 +20,7 @@ import ( type Options struct { ChunksPerNode int // number of chunks to upload per node GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageLabel string ReplicationFactorThreshold int // minimal replication factor per chunk Seed int64 @@ -32,7 +32,7 @@ func NewDefaultOptions() Options { return Options{ ChunksPerNode: 1, GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageLabel: "test-label", ReplicationFactorThreshold: 2, Seed: random.Int64(), @@ -90,7 +90,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int nodeName := sortedNodes[i] client := clients[nodeName] - batchID, err := client.CreatePostageBatch(ctx, o.PostageAmount, bee.MinimumBatchDepth, o.PostageLabel, false) + batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageTTL, bee.MinimumBatchDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: created batched id %w", nodeName, err) } diff --git a/pkg/check/pushsync/check_chunks.go b/pkg/check/pushsync/check_chunks.go index 9e79b4c45..1619f4164 100644 --- a/pkg/check/pushsync/check_chunks.go +++ b/pkg/check/pushsync/check_chunks.go @@ -42,7 +42,7 @@ func checkChunks(ctx context.Context, c orchestration.Cluster, o Options, l logg uploader := clients[nodeName] - batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", nodeName, err) } diff --git a/pkg/check/pushsync/check_lightnode.go b/pkg/check/pushsync/check_lightnode.go index a8dc8a0be..6f31e417e 100644 --- a/pkg/check/pushsync/check_lightnode.go +++ b/pkg/check/pushsync/check_lightnode.go @@ -34,7 +34,7 @@ func checkLightChunks(ctx context.Context, cluster orchestration.Cluster, o Opti // prepare postage batches for i := 0; i < len(lightNodes); i++ { nodeName := lightNodes[i] - batchID, err := clients[nodeName].GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := clients[nodeName].GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", nodeName, err) } @@ -46,7 +46,7 @@ func checkLightChunks(ctx context.Context, cluster orchestration.Cluster, o Opti nodeName := lightNodes[i] uploader := clients[nodeName] - batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", nodeName, err) } diff --git a/pkg/check/pushsync/pushsync.go b/pkg/check/pushsync/pushsync.go index 5045b8821..7942a65da 100644 --- a/pkg/check/pushsync/pushsync.go +++ b/pkg/check/pushsync/pushsync.go @@ -18,7 +18,7 @@ type Options struct { ChunksPerNode int GasPrice string Mode string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Retries int // number of reties on problems @@ -34,7 +34,7 @@ func NewDefaultOptions() Options { ChunksPerNode: 1, GasPrice: "", Mode: "default", - PostageAmount: 1000, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", Retries: 5, @@ -99,7 +99,7 @@ func (c *Check) defaultCheck(ctx context.Context, cluster orchestration.Cluster, nodeName := sortedNodes[i] client := clients[nodeName] - batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", nodeName, err) } diff --git a/pkg/check/redundancy/redundancy.go b/pkg/check/redundancy/redundancy.go index 0a7fc4d33..2181baa2a 100644 --- a/pkg/check/redundancy/redundancy.go +++ b/pkg/check/redundancy/redundancy.go @@ -20,18 +20,20 @@ import ( ) type Options struct { - PostageAmount int64 - PostageDepth uint64 - Seed int64 - DataSize int64 + DataSize int64 + PostageDepth uint64 + PostageLabel string + PostageTTL time.Duration + Seed int64 } func NewDefaultOptions() Options { return Options{ - PostageAmount: 1500000, - PostageDepth: 22, - Seed: time.Now().UnixNano(), - DataSize: 307200, + DataSize: 307200, + PostageDepth: 22, + PostageLabel: "test-label", + PostageTTL: 24 * time.Hour, + Seed: time.Now().UnixNano(), } } @@ -77,7 +79,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, o interf } c.logger.Infof("root hash: %s, chunks: %d", root.String(), len(chunks)) - batchID, err := uploadClient.GetOrCreateMutableBatch(ctx, opts.PostageAmount, opts.PostageDepth, "ci-redundancy") + batchID, err := uploadClient.GetOrCreateMutableBatch(ctx, opts.PostageTTL, opts.PostageDepth, opts.PostageLabel) if err != nil { return fmt.Errorf("get or create batch: %w", err) } diff --git a/pkg/check/retrieval/retrieval.go b/pkg/check/retrieval/retrieval.go index b091fa9b9..a4646e7f6 100644 --- a/pkg/check/retrieval/retrieval.go +++ b/pkg/check/retrieval/retrieval.go @@ -18,7 +18,7 @@ import ( // Options represents check options type Options struct { ChunksPerNode int // number of chunks to upload per node - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Seed int64 @@ -29,7 +29,7 @@ type Options struct { func NewDefaultOptions() Options { return Options{ ChunksPerNode: 1, - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageLabel: "test-label", PostageDepth: 16, Seed: random.Int64(), @@ -82,7 +82,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int downloadNodeIndex := (i + 1) % len(nodes) // download from the next node downloadNode := clients[nodes[downloadNodeIndex]] - batchID, err := uploadNode.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := uploadNode.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: created batched id %w", uploadNode.Name(), err) } diff --git a/pkg/check/settlements/settlements.go b/pkg/check/settlements/settlements.go index cb87d661b..fa1fa0ece 100644 --- a/pkg/check/settlements/settlements.go +++ b/pkg/check/settlements/settlements.go @@ -23,7 +23,7 @@ type Options struct { FileName string FileSize int64 GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Seed int64 @@ -41,7 +41,7 @@ func NewDefaultOptions() Options { FileName: "settlements", FileSize: 1 * 1024 * 1024, // 1mb GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageDepth: 20, PostageLabel: "test-label", Seed: 0, @@ -120,7 +120,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int client := clients[uNode] c.logger.Info("node", uNode) - batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", uNode, err) } diff --git a/pkg/check/smoke/load.go b/pkg/check/smoke/load.go index 44a15b53f..768b3954c 100644 --- a/pkg/check/smoke/load.go +++ b/pkg/check/smoke/load.go @@ -123,13 +123,13 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts var duration time.Duration c.logger.Infof("uploading to: %s", txName) - batchID, err := clients[txName].GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := clients[txName].GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { c.logger.Errorf("create new batch: %v", err) return } - c.logger.Info("using batch", "batch_id", batchID) + c.logger.WithField("batch_id", batchID).Info("using batch") address, duration, err = test.upload(ctx, txName, txData, batchID) if err != nil { diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index dd7557bce..eab7d7e65 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -20,7 +20,7 @@ import ( type Options struct { ContentSize int64 RndSeed int64 - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string TxOnErrWait time.Duration @@ -44,7 +44,7 @@ func NewDefaultOptions() Options { return Options{ ContentSize: 5000000, RndSeed: time.Now().UnixNano(), - PostageAmount: 50_000_000, + PostageTTL: 24 * time.Hour, PostageDepth: 24, PostageLabel: "test-label", TxOnErrWait: 10 * time.Second, @@ -160,14 +160,14 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int c.metrics.BatchCreateAttempts.Inc() - batchID, err = clients[txName].GetOrCreateMutableBatch(txCtx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err = clients[txName].GetOrCreateMutableBatch(txCtx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { c.logger.Errorf("create new batch: %v", err) c.metrics.BatchCreateErrors.Inc() continue } - c.logger.Info("using batch", "batch_id", batchID) + c.logger.WithField("batch_id", batchID).Info("using batch") c.metrics.UploadAttempts.Inc() address, txDuration, err = test.upload(txCtx, txName, txData, batchID) diff --git a/pkg/check/soc/soc.go b/pkg/check/soc/soc.go index 6c483c8a7..b27c4178c 100644 --- a/pkg/check/soc/soc.go +++ b/pkg/check/soc/soc.go @@ -21,7 +21,7 @@ import ( // Options represents check options type Options struct { GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string RequestTimeout time.Duration @@ -31,7 +31,7 @@ type Options struct { func NewDefaultOptions() Options { return Options{ GasPrice: "", - PostageAmount: 1, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", RequestTimeout: 5 * time.Minute, @@ -110,7 +110,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int id := hex.EncodeToString(idBytes) sig := hex.EncodeToString(signatureBytes) - batchID, err := node.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := node.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return fmt.Errorf("node %s: batch id %w", nodeName, err) } diff --git a/pkg/check/stake/contractutil.go b/pkg/check/stake/contractutil.go index 06ae166f0..ec588b592 100644 --- a/pkg/check/stake/contractutil.go +++ b/pkg/check/stake/contractutil.go @@ -55,7 +55,12 @@ func newSession(contract *Stake, geth *ethclient.Client, opts Options) (*StakeSe return nil, fmt.Errorf("get nonce: %w", err) } - auth, err := bind.NewKeyedTransactorWithChainID(privateKey, opts.GethChainID) + chainID, err := geth.ChainID(context.Background()) + if err != nil { + return nil, fmt.Errorf("get chain ID: %w", err) + } + + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, chainID) if err != nil { return nil, fmt.Errorf("new transactor: %w", err) } diff --git a/pkg/check/stake/stake.go b/pkg/check/stake/stake.go index a30953dc8..60157c93c 100644 --- a/pkg/check/stake/stake.go +++ b/pkg/check/stake/stake.go @@ -22,7 +22,6 @@ type Options struct { ContractAddr string CallerPrivateKey string GethURL string - GethChainID *big.Int } // NewDefaultOptions returns new default options @@ -30,7 +29,6 @@ func NewDefaultOptions() Options { return Options{ Amount: big.NewInt(100000000000000000), InsufficientAmount: big.NewInt(102400), - GethChainID: big.NewInt(12345), } } diff --git a/pkg/config/check.go b/pkg/config/check.go index b1748de0f..dd39eb169 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -54,7 +54,8 @@ type CheckType struct { // CheckGlobalConfig represents global configs for all checks type CheckGlobalConfig struct { - Seed int64 + Seed int64 + GethURL string } // Checks represents all available check types @@ -63,17 +64,17 @@ var Checks = map[string]CheckType{ NewAction: act.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - FileName *string `yaml:"file-name"` - FileSize *int64 `yaml:"file-size"` - PostageAmount *int64 `yaml:"postage-amount"` - PostageDepth *int64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - Seed *int64 `yaml:"seed"` + FileName *string `yaml:"file-name"` + FileSize *int64 `yaml:"file-size"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *int64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + Seed *int64 `yaml:"seed"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) } - opts := act.NewDefaultOptions() + opts := act.NewOptions() if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { return nil, fmt.Errorf("applying options: %w", err) @@ -89,7 +90,7 @@ var Checks = map[string]CheckType{ FileName *string `yaml:"file-name"` FileSize *int64 `yaml:"file-size"` GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` Seed *int64 `yaml:"seed"` @@ -130,15 +131,15 @@ var Checks = map[string]CheckType{ NewAction: fileretrieval.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - FileName *string `yaml:"file-name"` - FileSize *int64 `yaml:"file-size"` - FilesPerNode *int `yaml:"files-per-node"` - Full *bool `yaml:"full"` - GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` - PostageLabel *string `yaml:"postage-label"` - Seed *int64 `yaml:"seed"` - UploadNodeCount *int `yaml:"upload-node-count"` + FileName *string `yaml:"file-name"` + FileSize *int64 `yaml:"file-size"` + FilesPerNode *int `yaml:"files-per-node"` + Full *bool `yaml:"full"` + GasPrice *string `yaml:"gas-price"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageLabel *string `yaml:"postage-label"` + Seed *int64 `yaml:"seed"` + UploadNodeCount *int `yaml:"upload-node-count"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -216,13 +217,13 @@ var Checks = map[string]CheckType{ NewAction: manifest.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - FilesInCollection *int `yaml:"files-in-collection"` - GasPrice *string `yaml:"gas-price"` - MaxPathnameLength *int32 `yaml:"max-pathname-length"` - PostageAmount *int64 `yaml:"postage-amount"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - Seed *int64 `yaml:"seed"` + FilesInCollection *int `yaml:"files-in-collection"` + GasPrice *string `yaml:"gas-price"` + MaxPathnameLength *int32 `yaml:"max-pathname-length"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + Seed *int64 `yaml:"seed"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -256,7 +257,7 @@ var Checks = map[string]CheckType{ Count *int64 `yaml:"count"` AddressPrefix *int `yaml:"address-prefix"` GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` RequestTimeout *time.Duration `yaml:"request-timeout"` @@ -278,13 +279,13 @@ var Checks = map[string]CheckType{ NewAction: pullsync.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - ChunksPerNode *int `yaml:"chunks-per-node"` - GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` - PostageLabel *string `yaml:"postage-label"` - ReplicationFactorThreshold *int `yaml:"replication-factor-threshold"` - Seed *int64 `yaml:"seed"` - UploadNodeCount *int `yaml:"upload-node-count"` + ChunksPerNode *int `yaml:"chunks-per-node"` + GasPrice *string `yaml:"gas-price"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageLabel *string `yaml:"postage-label"` + ReplicationFactorThreshold *int `yaml:"replication-factor-threshold"` + Seed *int64 `yaml:"seed"` + UploadNodeCount *int `yaml:"upload-node-count"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -305,7 +306,7 @@ var Checks = map[string]CheckType{ ChunksPerNode *int `yaml:"chunks-per-node"` GasPrice *string `yaml:"gas-price"` Mode *string `yaml:"mode"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` Retries *int `yaml:"retries"` @@ -330,12 +331,12 @@ var Checks = map[string]CheckType{ NewAction: retrieval.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - ChunksPerNode *int `yaml:"chunks-per-node"` - PostageAmount *int64 `yaml:"postage-amount"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - Seed *int64 `yaml:"seed"` - UploadNodeCount *int `yaml:"upload-node-count"` + ChunksPerNode *int `yaml:"chunks-per-node"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + Seed *int64 `yaml:"seed"` + UploadNodeCount *int `yaml:"upload-node-count"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -358,7 +359,7 @@ var Checks = map[string]CheckType{ FileName *string `yaml:"file-name"` FileSize *int64 `yaml:"file-size"` GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` Seed *int64 `yaml:"seed"` @@ -384,8 +385,9 @@ var Checks = map[string]CheckType{ checkOpts := new(struct { ContentSize *int64 `yaml:"content-size"` RndSeed *int64 `yaml:"rnd-seed"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` TxOnErrWait *time.Duration `yaml:"tx-on-err-wait"` RxOnErrWait *time.Duration `yaml:"rx-on-err-wait"` NodesSyncWait *time.Duration `yaml:"nodes-sync-wait"` @@ -441,7 +443,7 @@ var Checks = map[string]CheckType{ NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { GasPrice *string `yaml:"gas-price"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` RequestTimeout *time.Duration `yaml:"request-timeout"` @@ -488,7 +490,6 @@ var Checks = map[string]CheckType{ ContractAddr *string `yaml:"contract-addr"` CallerPrivateKey *string `yaml:"private-key"` GethURL *string `yaml:"geth-url"` - GethChainID *big.Int `yaml:"geth-chain-id"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -527,8 +528,9 @@ var Checks = map[string]CheckType{ NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { RndSeed *int64 `yaml:"rnd-seed"` - PostageAmount *int64 `yaml:"postage-amount"` + PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` SleepDuration *time.Duration `yaml:"sleep-duration"` }) if err := check.Options.Decode(checkOpts); err != nil { @@ -567,10 +569,11 @@ var Checks = map[string]CheckType{ NewAction: redundancy.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - PostageAmount *int `yaml:"postage-amount"` - PostageDepth *int `yaml:"postage-depth"` - Seed *int `yaml:"seed"` - DataSize *int `yaml:"data-size"` + DataSize *int `yaml:"data-size"` + PostageDepth *int `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + Seed *int `yaml:"seed"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -606,9 +609,9 @@ var Checks = map[string]CheckType{ NewAction: gsoc.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - PostageAmount *int64 `yaml:"postage-amount"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -626,11 +629,11 @@ var Checks = map[string]CheckType{ NewAction: feed.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (interface{}, error) { checkOpts := new(struct { - PostageAmount *int64 `yaml:"postage-amount"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - NUpdates *int `yaml:"n-updates"` - RootRef *string `yaml:"root-ref"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + NUpdates *int `yaml:"n-updates"` + RootRef *string `yaml:"root-ref"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -673,6 +676,20 @@ func applyCheckConfig(global CheckGlobalConfig, local, opts interface{}) (err er ov.FieldByName(fieldName).Set(fieldValue) } } + case "GethURL": + if lv.Field(i).IsNil() { // set globally + if global.GethURL != "" { + v := reflect.ValueOf(global.GethURL) + ov.FieldByName(fieldName).Set(v) + } + } else { // set locally + fieldType := lt.Field(i).Type + fieldValue := lv.FieldByName(fieldName).Elem() + ft, ok := ot.FieldByName(fieldName) + if ok && fieldType.Elem().AssignableTo(ft.Type) { + ov.FieldByName(fieldName).Set(fieldValue) + } + } default: if lv.Field(i).IsNil() { fmt.Printf("field %s not set, using default value\n", fieldName) diff --git a/pkg/orchestration/cluster.go b/pkg/orchestration/cluster.go index da50f4a62..1a99915f9 100644 --- a/pkg/orchestration/cluster.go +++ b/pkg/orchestration/cluster.go @@ -23,21 +23,19 @@ type Cluster interface { FlattenSettlements(ctx context.Context) (settlements NodeGroupSettlements, err error) FlattenTopologies(ctx context.Context) (topologies map[string]bee.Topology, err error) FullNodeNames() (names []string) - ShuffledFullNodeClients(ctx context.Context, r *rand.Rand) ([]*bee.Client, error) GlobalReplicationFactor(ctx context.Context, a swarm.Address) (grf int, err error) LightNodeNames() (names []string) Name() string NodeGroup(name string) (ng NodeGroup, err error) NodeGroups() (l map[string]NodeGroup) - NodeGroupsSorted() (l []string) NodeNames() (names []string) Nodes() map[string]Node NodesClients(ctx context.Context) (map[string]*bee.Client, error) - NodesClientsAll(ctx context.Context) (map[string]*bee.Client, error) Overlays(ctx context.Context, exclude ...string) (overlays ClusterOverlays, err error) Peers(ctx context.Context, exclude ...string) (peers ClusterPeers, err error) RandomNode(ctx context.Context, r *rand.Rand) (node Node, err error) Settlements(ctx context.Context) (settlements ClusterSettlements, err error) + ShuffledFullNodeClients(ctx context.Context, r *rand.Rand) ([]*bee.Client, error) Size() (size int) Topologies(ctx context.Context) (topologies ClusterTopologies, err error) } diff --git a/pkg/orchestration/k8s/cluster.go b/pkg/orchestration/k8s/cluster.go index 7178a98ef..080d94564 100644 --- a/pkg/orchestration/k8s/cluster.go +++ b/pkg/orchestration/k8s/cluster.go @@ -4,13 +4,13 @@ import ( "context" "fmt" "math/rand" - "sort" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/beekeeper/pkg/bee" "github.com/ethersphere/beekeeper/pkg/logging" "github.com/ethersphere/beekeeper/pkg/orchestration" "github.com/ethersphere/beekeeper/pkg/orchestration/notset" + "github.com/ethersphere/beekeeper/pkg/swap" ) // compile check whether client implements interface @@ -35,6 +35,10 @@ func NewCluster(name string, o orchestration.ClusterOptions, log logging.Logger) no = newNodeOrchestrator(o.K8SClient, log) } + if o.SwapClient == nil { + o.SwapClient = &swap.NotSet{} + } + return &Cluster{ name: name, nodeOrchestrator: no, @@ -167,20 +171,6 @@ func (c *Cluster) NodeGroups() (l map[string]orchestration.NodeGroup) { return nodeGroups } -// NodeGroupsSorted returns sorted list of node names in the node group -func (c *Cluster) NodeGroupsSorted() (l []string) { - l = make([]string, len(c.nodeGroups)) - - i := 0 - for k := range c.nodeGroups { - l[i] = k - i++ - } - sort.Strings(l) - - return -} - // NodeGroup returns node group func (c *Cluster) NodeGroup(name string) (ng orchestration.NodeGroup, err error) { ng, ok := c.nodeGroups[name] @@ -263,17 +253,6 @@ func (c *Cluster) NodesClients(ctx context.Context) (map[string]*bee.Client, err return clients, nil } -// NodesClientsAll returns map of node's clients in the cluster -func (c *Cluster) NodesClientsAll(ctx context.Context) (map[string]*bee.Client, error) { - clients := make(map[string]*bee.Client) - for _, ng := range c.NodeGroups() { - for n, client := range ng.NodesClientsAll(ctx) { - clients[n] = client - } - } - return clients, nil -} - // Overlays returns ClusterOverlays excluding the provided node group names func (c *Cluster) Overlays(ctx context.Context, exclude ...string) (overlays orchestration.ClusterOverlays, err error) { overlays = make(orchestration.ClusterOverlays) diff --git a/pkg/orchestration/k8s/nodegroup.go b/pkg/orchestration/k8s/nodegroup.go index f68f960f9..ee3f91cc2 100644 --- a/pkg/orchestration/k8s/nodegroup.go +++ b/pkg/orchestration/k8s/nodegroup.go @@ -71,6 +71,7 @@ func (g *NodeGroup) AddNode(ctx context.Context, name string, inCluster bool, o APIURL: aURL, APIInsecureTLS: g.clusterOpts.APIInsecureTLS, Retry: 5, + SwapClient: g.clusterOpts.SwapClient, } for _, opt := range opts { @@ -488,11 +489,6 @@ func (g *NodeGroup) NodesClients(ctx context.Context) (map[string]*bee.Client, e return clients, nil } -// NodesClientsAll returns map of node's clients in the node group -func (g *NodeGroup) NodesClientsAll(ctx context.Context) map[string]*bee.Client { - return g.getClients() -} - // NodesSorted returns list of nodes sorted by names from the node group. func (g *NodeGroup) NodesSorted() []string { nodes := g.getNodes() @@ -506,11 +502,6 @@ func (g *NodeGroup) NodesSorted() []string { return l } -// Node returns node -func (g *NodeGroup) Node(name string) (orchestration.Node, error) { - return g.getNode(name) -} - // NodeClient returns node's client func (g *NodeGroup) NodeClient(name string) (*bee.Client, error) { return g.getClient(name) diff --git a/pkg/orchestration/nodegroup.go b/pkg/orchestration/nodegroup.go index c4898bf33..1b312395d 100644 --- a/pkg/orchestration/nodegroup.go +++ b/pkg/orchestration/nodegroup.go @@ -19,12 +19,10 @@ type NodeGroup interface { GetEthAddress(ctx context.Context, name string, o NodeOptions) (ethAddress string, err error) GroupReplicationFactor(ctx context.Context, a swarm.Address) (grf int, err error) Name() string - Node(name string) (Node, error) NodeClient(name string) (*bee.Client, error) NodeReady(ctx context.Context, name string) (ok bool, err error) Nodes() map[string]Node NodesClients(ctx context.Context) (map[string]*bee.Client, error) - NodesClientsAll(ctx context.Context) map[string]*bee.Client NodesSorted() (l []string) Overlays(ctx context.Context) (overlays NodeGroupOverlays, err error) Peers(ctx context.Context) (peers NodeGroupPeers, err error) diff --git a/pkg/simulate/retrieval/retrieval.go b/pkg/simulate/retrieval/retrieval.go index bb51686ae..c0bee66ae 100644 --- a/pkg/simulate/retrieval/retrieval.go +++ b/pkg/simulate/retrieval/retrieval.go @@ -18,7 +18,7 @@ import ( type Options struct { ChunksPerNode int // number of chunks to upload per node GasPrice string - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Seed int64 @@ -31,7 +31,7 @@ func NewDefaultOptions() Options { return Options{ ChunksPerNode: 1, GasPrice: "", - PostageAmount: 1000, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", Seed: random.Int64(), @@ -84,7 +84,7 @@ func (s *Simulation) Run(ctx context.Context, cluster orchestration.Cluster, opt nodeName := sortedNodes[i] client := clients[nodeName] - batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := client.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { s.logger.Infof("error: node %s: batch id %v", nodeName, err) continue diff --git a/pkg/simulate/upload/upload.go b/pkg/simulate/upload/upload.go index e933de2ba..f5ada2382 100644 --- a/pkg/simulate/upload/upload.go +++ b/pkg/simulate/upload/upload.go @@ -24,7 +24,7 @@ type Options struct { GasPrice string MaxFileSize int64 MinFileSize int64 - PostageAmount int64 + PostageTTL time.Duration PostageDepth uint64 PostageLabel string Retries int @@ -43,7 +43,7 @@ func NewDefaultOptions() Options { GasPrice: "", MaxFileSize: 1048576, // 1mb = 1*1024*1024 MinFileSize: 1048576, // 1mb = 1*1024*1024 - PostageAmount: 1000, + PostageTTL: 24 * time.Hour, PostageDepth: 16, PostageLabel: "test-label", Retries: 5, @@ -160,7 +160,7 @@ func (s *Simulation) Run(ctx context.Context, cluster orchestration.Cluster, opt return ctx.Err() } - batchID, err = c.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err = c.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { if errors.Is(ctx.Err(), context.DeadlineExceeded) { return nil diff --git a/pkg/stamper/node.go b/pkg/stamper/node.go index 9ce2e4cd9..209c96b57 100644 --- a/pkg/stamper/node.go +++ b/pkg/stamper/node.go @@ -24,25 +24,34 @@ func newNodeInfo(client *api.Client, name string, log logging.Logger) *node { } } -func (n *node) Create(ctx context.Context, amount uint64, depth uint16) error { - batchID, err := n.client.Postage.CreatePostageBatch(ctx, int64(amount), uint64(depth), "beekeeper") +func (n *node) Create(ctx context.Context, duration time.Duration, depth uint16, postageLabel string, secondsPerBlock int64) error { + price, err := n.getPrice(ctx) + if err != nil { + return err + } + + amount := (int64(duration.Seconds()) / secondsPerBlock) * price + + batchID, err := n.client.Postage.CreatePostageBatch(ctx, amount, uint64(depth), postageLabel) if err != nil { return fmt.Errorf("node %s: create postage batch: %w", n.name, err) } - n.log.Infof("node %s: created postage batch %s", n.name, batchID) + bzzPrice := float64(amount*int64(1< 0 && !slices.Contains(batchIDs, batch.BatchID) { + if len(opts.batchIDs) > 0 && !slices.Contains(opts.batchIDs, batch.BatchID) { + return false + } + + if len(opts.postageLabels) > 0 && !slices.Contains(opts.postageLabels, batch.Label) { return false } diff --git a/pkg/stamper/stamper.go b/pkg/stamper/stamper.go index f3184ee12..7263c4915 100644 --- a/pkg/stamper/stamper.go +++ b/pkg/stamper/stamper.go @@ -7,6 +7,7 @@ import ( "net/url" "time" + "github.com/ethersphere/bee/v2/pkg/postage" "github.com/ethersphere/beekeeper/pkg/bee" "github.com/ethersphere/beekeeper/pkg/bee/api" "github.com/ethersphere/beekeeper/pkg/k8s" @@ -17,12 +18,19 @@ import ( type Option func(*options) type options struct { - batchIds []string + batchIDs []string + postageLabels []string } func WithBatchIDs(batchIds []string) Option { return func(o *options) { - o.batchIds = batchIds + o.batchIDs = batchIds + } +} + +func WithPostageLabels(postageLabels []string) Option { + return func(o *options) { + o.postageLabels = postageLabels } } @@ -55,6 +63,10 @@ func New(cfg *ClientConfig) *Client { cfg.Log = logging.New(io.Discard, 0) } + if cfg.SwapClient == nil { + cfg.SwapClient = &swap.NotSet{} + } + return &Client{ log: cfg.Log, namespace: cfg.Namespace, @@ -67,10 +79,18 @@ func New(cfg *ClientConfig) *Client { } // Create creates a postage batch. -func (s *Client) Create(ctx context.Context, amount uint64, depth uint16) error { +func (s *Client) Create(ctx context.Context, duration time.Duration, depth uint16, postageLabel string) error { + if duration == 0 { + return fmt.Errorf("duration must be greater than 0") + } + + if depth <= postage.BucketDepth { + return fmt.Errorf("depth must be greater than %d", postage.BucketDepth) + } + s.log.WithFields(map[string]interface{}{ - "amount": amount, - "depth": depth, + "duration": duration, + "depth": depth, }).Infof("creating postage batch on nodes") nodes, err := s.getNodes(ctx) @@ -78,8 +98,13 @@ func (s *Client) Create(ctx context.Context, amount uint64, depth uint16) error return fmt.Errorf("get nodes: %w", err) } + blockTime, err := s.swapClient.FetchBlockTime(ctx, swap.WithOffset(1000)) + if err != nil { + return fmt.Errorf("fetching block time: %w", err) + } + for _, node := range nodes { - if err := node.Create(ctx, amount, depth); err != nil { + if err := node.Create(ctx, duration, depth, postageLabel, blockTime); err != nil { s.log.Errorf("node %s create postage batch: %v", node.name, err) } } @@ -100,7 +125,7 @@ func (s *Client) Dilute(ctx context.Context, usageThreshold float64, dilutionDep } for _, node := range nodes { - if err := node.Dilute(ctx, usageThreshold, dilutionDepth, processOptions(opts...).batchIds); err != nil { + if err := node.Dilute(ctx, usageThreshold, dilutionDepth, processOptions(opts...)); err != nil { s.log.Errorf("node %s dilute postage batch: %v", node.name, err) } } @@ -122,13 +147,13 @@ func (s *Client) Set(ctx context.Context, ttlThreshold time.Duration, topupTo ti return fmt.Errorf("get nodes: %w", err) } - blockTime, err := s.swapClient.FetchBlockTime(ctx) + blockTime, err := s.swapClient.FetchBlockTime(ctx, swap.WithOffset(1000)) if err != nil { return fmt.Errorf("fetching block time: %w", err) } for _, node := range nodes { - if err := node.Set(ctx, ttlThreshold, topupTo, usageThreshold, dilutionDepth, blockTime, processOptions(opts...).batchIds); err != nil { + if err := node.Set(ctx, ttlThreshold, topupTo, usageThreshold, dilutionDepth, blockTime, processOptions(opts...)); err != nil { s.log.Errorf("node %s set postage batch: %v", node.name, err) } } @@ -148,13 +173,13 @@ func (s *Client) Topup(ctx context.Context, ttlThreshold time.Duration, topupTo return fmt.Errorf("get nodes: %w", err) } - blockTime, err := s.swapClient.FetchBlockTime(ctx) + blockTime, err := s.swapClient.FetchBlockTime(ctx, swap.WithOffset(1000)) if err != nil { return fmt.Errorf("fetching block time: %w", err) } for _, node := range nodes { - if err := node.Topup(ctx, ttlThreshold, topupTo, blockTime, processOptions(opts...).batchIds); err != nil { + if err := node.Topup(ctx, ttlThreshold, topupTo, blockTime, processOptions(opts...)); err != nil { s.log.Errorf("node %s topup postage batch: %v", node.name, err) } } diff --git a/pkg/swap/block.go b/pkg/swap/block.go index d90d3f2df..001e214ea 100644 --- a/pkg/swap/block.go +++ b/pkg/swap/block.go @@ -2,12 +2,45 @@ package swap import ( "context" + "errors" "fmt" + "math" "net/http" "strconv" ) -func (g *GethClient) FetchBlockTime(ctx context.Context) (int64, error) { +var ( + ErrEmptyTimestamp = errors.New("empty timestamp, offset too large") + ErrInvalidTimestamp = errors.New("invalid timestamp") + ErrEmptyResult = errors.New("empty result") + ErrInvalidResult = errors.New("invalid result") +) + +type Option func(*options) + +type options struct { + offset int64 +} + +func WithOffset(offset int64) Option { + return func(o *options) { + if offset > 0 { + o.offset = offset + } else { + o.offset = 1 + } + } +} + +// FetchBlockTime estimates the average block time by comparing timestamps +// of the latest block and an earlier block, adjusting the offset if needed. +func (g *GethClient) FetchBlockTime(ctx context.Context, opts ...Option) (int64, error) { + o := processOptions(opts...) + + retryOffset := o.offset + + var err error + latestBlockNumber, err := g.fetchLatestBlockNumber(ctx) if err != nil { return 0, fmt.Errorf("fetch latest block number: %w", err) @@ -18,16 +51,35 @@ func (g *GethClient) FetchBlockTime(ctx context.Context) (int64, error) { return 0, fmt.Errorf("fetch latest block timestamp: %w", err) } - timestampPrevious, err := g.fetchBlockTimestamp(ctx, latestBlockNumber-1) - if err != nil { - return 0, fmt.Errorf("fetch previous block timestamp: %w", err) + // limit retryOffset to at most half of the latest block number + if retryOffset > latestBlockNumber/2 { + retryOffset = latestBlockNumber / 2 + g.logger.Warningf("offset too large, reduced to %d", retryOffset) + } + + var timestampPrevious int64 + + for retryOffset >= 1 { + blockNumber := latestBlockNumber - retryOffset + timestampPrevious, err = g.fetchBlockTimestamp(ctx, blockNumber) + if err == nil { + break + } + if !errors.Is(err, ErrEmptyTimestamp) || retryOffset == 1 { + return 0, fmt.Errorf("fetch previous block timestamp (block %d): %w", blockNumber, err) + } + + // reduce offset for next attempt, ensuring it remains >= 1 + retryOffset = int64(math.Max(1, float64(retryOffset)/2)) + g.logger.Warningf("%v at block %d, offset reduced to %d and retrying...", err, blockNumber, retryOffset) } - blockTime := timestampLatest - timestampPrevious + blockTime := float64(timestampLatest-timestampPrevious) / float64(retryOffset) + roundedBlockTime := int64(math.Round(blockTime)) - g.logger.Tracef("block time: %d seconds", blockTime) + g.logger.Tracef("avg block time for last %d blocks: %f, using rounded seconds: %d", retryOffset, blockTime, roundedBlockTime) - return blockTime, nil + return roundedBlockTime, nil } type rpcRequest struct { @@ -55,11 +107,11 @@ func (g *GethClient) fetchLatestBlockNumber(ctx context.Context) (int64, error) } if len(resp.Result) == 0 { - return 0, fmt.Errorf("empty result") + return 0, ErrEmptyResult } if resp.Result[:2] != "0x" { - return 0, fmt.Errorf("invalid result") + return 0, ErrInvalidResult } blockNumber, err := strconv.ParseInt(resp.Result[2:], 16, 64) @@ -91,12 +143,22 @@ func (g *GethClient) fetchBlockTimestamp(ctx context.Context, blockNumber int64) } if len(resp.Result.Timestamp) == 0 { - return 0, fmt.Errorf("empty timestamp") + return 0, ErrEmptyTimestamp } if resp.Result.Timestamp[:2] != "0x" { - return 0, fmt.Errorf("invalid timestamp") + return 0, ErrInvalidTimestamp } return strconv.ParseInt(resp.Result.Timestamp[2:], 16, 64) } + +func processOptions(opts ...Option) *options { + o := &options{ + offset: 1, + } + for _, opt := range opts { + opt(o) + } + return o +} diff --git a/pkg/swap/notset.go b/pkg/swap/notset.go index 91b6fb630..0edd15ed1 100644 --- a/pkg/swap/notset.go +++ b/pkg/swap/notset.go @@ -6,7 +6,7 @@ import ( ) // ErrNotSet represents error when Swap client is not set -var ErrNotSet = errors.New("swap client not set") +var ErrNotSet = errors.New("swap client not initialized because geth-url is missing") // compile check whether NotSet implements Swap interface var _ Client = (*NotSet)(nil) @@ -33,6 +33,6 @@ func (n *NotSet) AttestOverlayEthAddress(ctx context.Context, ethAddr string) (t } // FetchBlockTime(ctx context.Context) (blockTime int64, err error) -func (n *NotSet) FetchBlockTime(ctx context.Context) (blockTime int64, err error) { +func (n *NotSet) FetchBlockTime(ctx context.Context, opts ...Option) (blockTime int64, err error) { return 0, ErrNotSet } diff --git a/pkg/swap/swap.go b/pkg/swap/swap.go index 2a589ebfa..12c9deba9 100644 --- a/pkg/swap/swap.go +++ b/pkg/swap/swap.go @@ -20,5 +20,5 @@ type Client interface { SendBZZ(ctx context.Context, to string, amount float64) (tx string, err error) SendGBZZ(ctx context.Context, to string, amount float64) (tx string, err error) AttestOverlayEthAddress(ctx context.Context, ethAddr string) (tx string, err error) - FetchBlockTime(ctx context.Context) (blockTime int64, err error) + FetchBlockTime(ctx context.Context, opts ...Option) (blockTime int64, err error) } diff --git a/pkg/test/case.go b/pkg/test/case.go index 56ab57254..4ef426d04 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -3,6 +3,7 @@ package bee import ( "context" "math/rand" + "time" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/beekeeper/pkg/bee" @@ -25,13 +26,13 @@ type CheckCase struct { } type CaseOptions struct { - FileName string - FileSize int64 - GasPrice string - PostageAmount int64 - PostageLabel string - Seed int64 - PostageDepth uint64 + FileName string + FileSize int64 + GasPrice string + PostageTTL time.Duration + PostageLabel string + Seed int64 + PostageDepth uint64 } func NewCheckCase(ctx context.Context, cluster orchestration.Cluster, caseOpts CaseOptions, logger logging.Logger) (*CheckCase, error) { diff --git a/pkg/test/node.go b/pkg/test/node.go index 66c889541..acf48a5a9 100644 --- a/pkg/test/node.go +++ b/pkg/test/node.go @@ -46,7 +46,7 @@ func (b *BeeV2) UploadRandomFile(ctx context.Context) (File, error) { func (b *BeeV2) UploadFile(ctx context.Context, file File) error { depth := 2 + bee.EstimatePostageBatchDepth(b.opts.FileSize) - batchID, err := b.client.CreatePostageBatch(ctx, b.opts.PostageAmount, depth, b.opts.PostageLabel, false) + batchID, err := b.client.GetOrCreateMutableBatch(ctx, b.opts.PostageTTL, depth, b.opts.PostageLabel) if err != nil { return fmt.Errorf("node %s: created batch id %w", b.name, err) } @@ -79,7 +79,7 @@ func (b *BeeV2) ExpectToHaveFile(ctx context.Context, file File) error { func (b *BeeV2) NewChunkUploader(ctx context.Context) (*ChunkUploader, error) { o := b.opts - batchID, err := b.client.GetOrCreateMutableBatch(ctx, o.PostageAmount, o.PostageDepth, o.PostageLabel) + batchID, err := b.client.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { return nil, fmt.Errorf("node %s: batch id %w", b.name, err) } diff --git a/version.go b/version.go index aaf618521..a5d3f36b9 100644 --- a/version.go +++ b/version.go @@ -1,9 +1,8 @@ package beekeeper var ( - version = "0.25.0" // manually set semantic version number - - commit string // automatically set git commit hash + version string // automatically set semantic version number + commit string // automatically set git commit hash // Version TODO Version = func() string {