Add more tools

go.mod

@@ -6,6 +6,7 @@ replace github.com/goccy/go-graphviz => github.com/alexsporn/go-graphviz v0.0.0-
 
 require (
 	github.com/VictoriaMetrics/fastcache v1.12.2
+	github.com/dustin/go-humanize v1.0.1
 	github.com/fjl/memsize v0.0.2
 	github.com/goccy/go-graphviz v0.1.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -26,6 +27,7 @@ require (
 	github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20240425095808-113b21573349
 	github.com/iotaledger/hive.go/sql v0.0.0-20240425095808-113b21573349
 	github.com/iotaledger/hive.go/stringify v0.0.0-20240425095808-113b21573349
+	github.com/iotaledger/hive.go/web v0.0.0-20240425095808-113b21573349
 	github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240425100742-5c85b6d16701
 	github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240425100432-05e1bf8fc089
 	github.com/iotaledger/iota.go/v4 v4.0.0-20240503105040-c86882e71808
@@ -45,7 +47,8 @@ require (
 	github.com/zyedidia/generic v1.2.1
 	go.uber.org/atomic v1.11.0
 	go.uber.org/dig v1.17.1
-	golang.org/x/crypto v0.22.0
+	golang.org/x/crypto v0.23.0
+	golang.org/x/term v0.20.0
 	google.golang.org/grpc v1.63.2
 	google.golang.org/protobuf v1.33.0
 	gorm.io/gorm v1.25.9
@@ -63,7 +66,6 @@ require (
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/eclipse/paho.mqtt.golang v1.4.3 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/ethereum/go-ethereum v1.14.0 // indirect
@@ -183,8 +185,8 @@ require (
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect

go.sum

@@ -321,6 +321,8 @@ github.com/iotaledger/hive.go/sql v0.0.0-20240425095808-113b21573349 h1:+bHqh0gC
 github.com/iotaledger/hive.go/sql v0.0.0-20240425095808-113b21573349/go.mod h1:nEwIUQIvNMUs2DwM2870Er3foVQTzwPDUtzEKy+evGg=
 github.com/iotaledger/hive.go/stringify v0.0.0-20240425095808-113b21573349 h1:9cuEF+WvxB/xBLkQu6H3/pHYE5KAqY98oniUFYezvzc=
 github.com/iotaledger/hive.go/stringify v0.0.0-20240425095808-113b21573349/go.mod h1:O4p7UmsfoeLqtAUwrKbq0lXMxjY/MLQSpZSavvvvGig=
+github.com/iotaledger/hive.go/web v0.0.0-20240425095808-113b21573349 h1:LrLWPsUcbZjR3kF5NJbuou84l+aCm8dPx4S5pQ/9bJ0=
+github.com/iotaledger/hive.go/web v0.0.0-20240425095808-113b21573349/go.mod h1:YhGg68kOBM083FzmxjhQD3n/8AeG/4xr4fgM6qq1q9I=
 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240425100742-5c85b6d16701 h1:YWIaqOp7+DjxG4O6797Uw+K8rPRJN56rbwX6KpWsvEc=
 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240425100742-5c85b6d16701/go.mod h1:oqSLTV1hlpdLdi0MWt39c+EFgERdPM34ACAYM+4g3Oc=
 github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240425100432-05e1bf8fc089 h1:+NRPSbH6tkop8p+MhjCR9nvs8ng3Oo2yju5FXcDEkBY=
@@ -750,8 +752,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
@@ -854,18 +856,20 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/toolset/benchmark.go

@@ -0,0 +1,175 @@
+package toolset
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	flag "github.com/spf13/pflag"
+
+	"github.com/iotaledger/hive.go/app/configuration"
+	"github.com/iotaledger/hive.go/db"
+	"github.com/iotaledger/hive.go/kvstore"
+	"github.com/iotaledger/iota-core/pkg/storage/database"
+	iotago_tpkg "github.com/iotaledger/iota.go/v4/tpkg"
+)
+
+// estimateRemainingTime estimates the remaining time for a running operation and returns the finished percentage.
+func estimateRemainingTime(timeStart time.Time, current int64, total int64) (percentage float64, remaining time.Duration) {
+	ratio := float64(current) / float64(total)
+	totalTime := time.Duration(float64(time.Since(timeStart)) / ratio)
+	remaining = time.Until(timeStart.Add(totalTime))
+
+	return ratio * 100.0, remaining
+}
+
+func benchmarkIO(args []string) error {
+	fs := configuration.NewUnsortedFlagSet("", flag.ContinueOnError)
+	objectsCountFlag := fs.Int(FlagToolBenchmarkCount, 500000, "objects count")
+	objectsSizeFlag := fs.Int(FlagToolBenchmarkSize, 1000, "objects size in bytes")
+
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", ToolBenchmarkIO)
+		fs.PrintDefaults()
+		println(fmt.Sprintf("\nexample: %s --%s %d --%s %d",
+			ToolBenchmarkIO,
+			FlagToolBenchmarkCount,
+			500000,
+			FlagToolBenchmarkSize,
+			1000))
+	}
+
+	if err := parseFlagSet(fs, args); err != nil {
+		return err
+	}
+
+	objectCnt := *objectsCountFlag
+	size := *objectsSizeFlag
+
+	tempDir, err := os.MkdirTemp("", "benchmarkIO")
+	if err != nil {
+		return fmt.Errorf("can't create temp dir: %w", err)
+	}
+
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	store, err := database.StoreWithDefaultSettings(tempDir, true, db.EngineRocksDB, db.EngineRocksDB)
+	if err != nil {
+		return fmt.Errorf("database initialization failed: %w", err)
+	}
+
+	batchWriter := kvstore.NewBatchedWriter(store)
+	writeDoneWaitGroup := &sync.WaitGroup{}
+	writeDoneWaitGroup.Add(objectCnt)
+
+	ts := time.Now()
+
+	lastStatusTime := time.Now()
+	for i := 0; i < objectCnt; i++ {
+		// one read operation and one write operation per cycle
+		batchWriter.Enqueue(newBenchmarkObject(store, writeDoneWaitGroup, iotago_tpkg.RandBytes(32), iotago_tpkg.RandBytes(size)))
+
+		if time.Since(lastStatusTime) >= printStatusInterval {
+			lastStatusTime = time.Now()
+
+			duration := time.Since(ts)
+			bytes := uint64(i * (32 + size))
+			totalBytes := uint64(objectCnt * (32 + size))
+			bytesPerSecond := uint64(float64(bytes) / duration.Seconds())
+			objectsPerSecond := uint64(float64(i) / duration.Seconds())
+			percentage, remaining := estimateRemainingTime(ts, int64(i), int64(objectCnt))
+			fmt.Printf("Average IO speed: %s/s (%dx 32+%d byte chunks, total %s/%s, %d objects/s, %0.2f%%. %v left ...)\n", humanize.Bytes(bytesPerSecond), i, size, humanize.Bytes(bytes), humanize.Bytes(totalBytes), objectsPerSecond, percentage, remaining.Truncate(time.Second))
+		}
+	}
+
+	writeDoneWaitGroup.Wait()
+
+	if err := store.Flush(); err != nil {
+		return fmt.Errorf("flush database failed: %w", err)
+	}
+
+	if err := store.Close(); err != nil {
+		return fmt.Errorf("close database failed: %w", err)
+	}
+
+	te := time.Now()
+	duration := te.Sub(ts)
+	totalBytes := uint64(objectCnt * (32 + size))
+	bytesPerSecond := uint64(float64(totalBytes) / duration.Seconds())
+	objectsPerSecond := uint64(float64(objectCnt) / duration.Seconds())
+
+	fmt.Printf("Average IO speed: %s/s (%dx 32+%d byte chunks, total %s/%s, %d objects/s, took %v)\n", humanize.Bytes(bytesPerSecond), objectCnt, size, humanize.Bytes(totalBytes), humanize.Bytes(totalBytes), objectsPerSecond, duration.Truncate(time.Millisecond))
+
+	return nil
+}
+
+func benchmarkCPU(args []string) error {
+	fs := configuration.NewUnsortedFlagSet("", flag.ContinueOnError)
+	cpuThreadsFlag := fs.Int(FlagToolBenchmarkThreads, runtime.NumCPU(), "thread count")
+	durationFlag := fs.Duration(FlagToolBenchmarkDuration, 1*time.Minute, "duration")
+
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", ToolBenchmarkCPU)
+		fs.PrintDefaults()
+		println(fmt.Sprintf("\nexample: %s --%s %d --%s 1m0s",
+			ToolBenchmarkCPU,
+			FlagToolBenchmarkThreads,
+			2,
+			FlagToolBenchmarkDuration))
+	}
+
+	if err := parseFlagSet(fs, args); err != nil {
+		return err
+	}
+
+	threads := *cpuThreadsFlag
+	duration := *durationFlag
+
+	benchmarkCtx, benchmarkCtxCancel := context.WithTimeout(context.Background(), duration)
+	defer benchmarkCtxCancel()
+
+	ts := time.Now()
+
+	// doBenchmarkCPU mines with blake2b until the context has been canceled.
+	// it returns the number of calculated hashes.
+	doBenchmarkCPU := func(ctx context.Context, numWorkers int) uint64 {
+		var counter uint64
+		var wg sync.WaitGroup
+
+		// random digest
+		powDigest := iotago_tpkg.RandBytes(32)
+
+		go func() {
+			for ctx.Err() == nil {
+				time.Sleep(printStatusInterval)
+
+				elapsed := time.Since(ts)
+				percentage, remaining := estimateRemainingTime(ts, elapsed.Milliseconds(), duration.Milliseconds())
+				megahashesPerSecond := float64(counter) / (elapsed.Seconds() * 1000000)
+				fmt.Printf("Average CPU speed: %0.2fMH/s (%d thread(s), %0.2f%%. %v left ...)\n", megahashesPerSecond, numWorkers, percentage, remaining.Truncate(time.Second))
+			}
+		}()
+
+		for i := 0; i < numWorkers; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+
+				cpuBenchmarkWorker(ctx, powDigest, &counter)
+			}()
+		}
+		wg.Wait()
+
+		return counter
+	}
+
+	hashes := doBenchmarkCPU(benchmarkCtx, threads)
+	megahashesPerSecond := float64(hashes) / (duration.Seconds() * 1000000)
+	fmt.Printf("Average CPU speed: %0.2fMH/s (%d thread(s), took %v)\n", megahashesPerSecond, threads, duration.Truncate(time.Millisecond))
+
+	return nil
+}

pkg/toolset/benchmark_cpu.go

@@ -0,0 +1,28 @@
+package toolset
+
+import (
+	"context"
+	"crypto"
+	"sync/atomic"
+
+	"golang.org/x/crypto/blake2b"
+)
+
+const (
+	// Hash defines the hash function that is used to compute the PoW digest.
+	Hash = crypto.BLAKE2b_256
+)
+
+func cpuBenchmarkWorker(ctx context.Context, powDigest []byte, counter *uint64) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+
+		default:
+			result := blake2b.Sum256(powDigest)
+			powDigest = result[:]
+			atomic.AddUint64(counter, 1)
+		}
+	}
+}

pkg/toolset/benchmark_io.go

@@ -0,0 +1,49 @@
+package toolset
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/iotaledger/hive.go/kvstore"
+	iotago_tpkg "github.com/iotaledger/iota.go/v4/tpkg"
+)
+
+type benchmarkObject struct {
+	store              kvstore.KVStore
+	writeDoneWaitGroup *sync.WaitGroup
+	key                []byte
+	value              []byte
+}
+
+func newBenchmarkObject(store kvstore.KVStore, writeDoneWaitGroup *sync.WaitGroup, key []byte, value []byte) *benchmarkObject {
+	return &benchmarkObject{
+		store:              store,
+		writeDoneWaitGroup: writeDoneWaitGroup,
+		key:                key,
+		value:              value,
+	}
+}
+
+func (bo *benchmarkObject) BatchWrite(batchedMuts kvstore.BatchedMutations) {
+	if err := batchedMuts.Set(bo.key, bo.value); err != nil {
+		panic(fmt.Errorf("write operation failed: %w", err))
+	}
+}
+
+func (bo *benchmarkObject) BatchWriteDone() {
+	// do a read operation after the batchwrite is done,
+	// so the write and read operations are equally distributed over the whole benchmark run.
+	if _, err := bo.store.Has(iotago_tpkg.RandBytes(32)); err != nil {
+		panic(fmt.Errorf("read operation failed: %w", err))
+	}
+
+	bo.writeDoneWaitGroup.Done()
+}
+
+func (bo *benchmarkObject) BatchWriteScheduled() bool {
+	return false
+}
+
+func (bo *benchmarkObject) ResetBatchWriteScheduled() {
+	// do nothing
+}