diff --git a/astrobwt/astrobwt_fast/astrobwt_optimized.go b/astrobwt/astrobwt_fast/astrobwt_optimized.go new file mode 100644 index 00000000..b215121a --- /dev/null +++ b/astrobwt/astrobwt_fast/astrobwt_optimized.go @@ -0,0 +1,162 @@ +package astrobwt_fast + +import "unsafe" +import "hash" +import "sync" +import "crypto/rand" +import "encoding/binary" +import "golang.org/x/crypto/sha3" +import "golang.org/x/crypto/salsa20/salsa" + +const stage1_length uint32 = 9973 // it is a prime + +// see here to improve the algorithms more https://github.com/y-256/libdivsufsort/blob/wiki/SACA_Benchmarks.md +// this optimized algorithm is used only in the miner and not in the blockchain + +type ScratchData struct { + hasher hash.Hash + stage1 [stage1_length + 64]byte // 10 KB stages are taken from it + stage1_result *[stage1_length + 1]uint16 + stage1_result_bytes *[(stage1_length) * 2]uint8 + indices [stage1_length + 1]uint32 // 40 KB + tmp_indices [stage1_length + 1]uint32 // 40 KB +} + +var Pool = sync.Pool{New: func() interface{} { + var d ScratchData + d.hasher = sha3.New256() + d.stage1_result = ((*[stage1_length + 1]uint16)(unsafe.Pointer(&d.indices[0]))) + d.stage1_result_bytes = ((*[(stage1_length) * 2]byte)(unsafe.Pointer(&d.indices[0]))) + + return &d +}} + +func POW_optimized(inputdata []byte, data *ScratchData) (outputhash [32]byte) { + + defer func() { + if r := recover(); r != nil { // if something happens due to RAM issues in miner, we should continue, system will crash sooner or later + var buf [16]byte + rand.Read(buf[:]) + outputhash = sha3.Sum256(buf[:]) // return a falsified has which will fail the check + } + }() + + var key [32]byte + for i := range data.stage1 { + data.stage1[i] = 0 + } + + var counter [16]byte + + data.hasher.Reset() + data.hasher.Write(inputdata) + _ = data.hasher.Sum(key[:0]) + + salsa.XORKeyStream(data.stage1[:stage1_length], data.stage1[:stage1_length], &counter, &key) + sort_indices(stage1_length, data.stage1[:stage1_length+40], data.stage1_result[:], data) // extra 40 bytes since we may read them, but we never write them + + if LittleEndian { + data.hasher.Reset() + data.hasher.Write(data.stage1_result_bytes[:]) + _ = data.hasher.Sum(key[:0]) + } else { + var s [stage1_length * 2]byte + for i, c := range data.stage1_result { + binary.LittleEndian.PutUint16(s[i<<1:], c) + } + data.hasher.Reset() + data.hasher.Write(s[:]) + _ = data.hasher.Sum(key[:0]) + } + + copy(outputhash[:], key[:]) + return +} + +func fix(v []byte, indices []uint32, i int) { + prev_t := indices[i] + t := indices[i+1] + + data_a := binary.BigEndian.Uint32(v[((t)&0xffff)+2:]) + if data_a < binary.BigEndian.Uint32(v[((prev_t)&0xffff)+2:]) { + t2 := prev_t + j := i + _ = indices[j+1] + for { + indices[j+1] = prev_t + j-- + if j < 0 { + break + } + prev_t = indices[j] + if (t^prev_t) <= 0xffff && data_a < binary.BigEndian.Uint32(v[((prev_t)&0xffff)+2:]) { + continue + } else { + break + } + } + indices[j+1] = t + t = t2 + } +} + +// basically +func sort_indices(N uint32, v []byte, output []uint16, d *ScratchData) { + + var byte_counters [2][256]byte + var counters [2][256]uint16 + + v[N] = 0 // make sure extra byte accessed is zero + + indices := d.indices[:] + tmp_indices := d.tmp_indices[:] + + for _, c := range v[:N] { + byte_counters[1][c]++ + } + byte_counters[0] = byte_counters[1] + byte_counters[0][v[0]]-- + + counters[0][0] = uint16(byte_counters[0][0]) + counters[1][0] = uint16(byte_counters[1][0]) - 1 + + c0 := counters[0][0] + c1 := counters[1][0] + + for i := 1; i < 256; i++ { + c0 += uint16(byte_counters[0][i]) + c1 += uint16(byte_counters[1][i]) + + counters[0][i] = c0 + counters[1][i] = c1 + } + + counters0 := counters[0][:] + for i := int(N); i >= 1; i-- { + byte0 := uint32(v[i-1]) + byte1 := uint32(v[i]) // here we can access extra byte from input array so make sure its zero + tmp_indices[counters0[v[i]]] = byte0<<24 | byte1<<16 | uint32(i-1) + counters0[v[i]]-- + } + + counters1 := counters[1][:] + _ = tmp_indices[N-1] + for i := int(N - 1); i >= 0; i-- { + data := tmp_indices[i] + tmp := counters1[data>>24] + counters1[data>>24]-- + indices[tmp] = data + } + + for i := 1; i < int(N); i++ { // no BC here + if indices[i-1]&0xffff0000 == indices[i]&0xffff0000 { + fix(v, indices, i-1) + } + } + + // after fixing, convert indices to output + _ = output[N] + for i, c := range indices[:N] { + output[i] = uint16(c) + } +} diff --git a/astrobwt/astrobwt_fast/astrobwt_optimized_test.go b/astrobwt/astrobwt_fast/astrobwt_optimized_test.go new file mode 100644 index 00000000..0e324dc9 --- /dev/null +++ b/astrobwt/astrobwt_fast/astrobwt_optimized_test.go @@ -0,0 +1,50 @@ +package astrobwt_fast + +import "crypto/rand" + +//import "strings" +import "testing" + +//import "encoding/hex" + +import "github.com/deroproject/derohe/astrobwt" + +func TestPOW_optimized_v1(t *testing.T) { + scratch := Pool.Get().(*ScratchData) + + for i := 0; i < 4000; i++ { + buf := make([]byte, 400, 400) + rand.Read(buf) + + expected_output := astrobwt.POW16(buf[:]) + actual_output := POW_optimized(buf[:], scratch) + + if string(expected_output[:]) != string(actual_output[:]) { + t.Fatalf("Test failed: POW and POW_optimized returns different for i=%d buf %x", i, buf) + } + + } +} + +func BenchmarkFastSA(t *testing.B) { + var buf [stage1_length + 40]byte + rand.Read(buf[:]) + var output [10000]uint16 + + scratch := Pool.Get().(*ScratchData) + + rand.Read(buf[:stage1_length]) + for i := 0; i < t.N; i++ { + sort_indices(stage1_length, buf[:], output[:], scratch) + // t.Logf("Ran algo %d", i) + } +} + +func BenchmarkPOW_optimized(t *testing.B) { + var buf [128]byte + rand.Read(buf[:]) + scratch := Pool.Get().(*ScratchData) + for i := 0; i < t.N; i++ { + _ = POW_optimized(buf[:], scratch) + } +} diff --git a/astrobwt/astrobwt_fast/endian_big.go b/astrobwt/astrobwt_fast/endian_big.go new file mode 100644 index 00000000..f124493f --- /dev/null +++ b/astrobwt/astrobwt_fast/endian_big.go @@ -0,0 +1,9 @@ +//go:build armbe || arm64be || mips || mips64 || ppc64 || s390 || s390x || sparc || sparc64 +// +build armbe arm64be mips mips64 ppc64 s390 s390x sparc sparc64 + +package astrobwt_fast + +const LittleEndian = false +const BigEndian = true + +// see https://github.com/golang/go/blob/master/src/go/build/syslist.go diff --git a/astrobwt/astrobwt_fast/endian_little.go b/astrobwt/astrobwt_fast/endian_little.go new file mode 100644 index 00000000..78ba923e --- /dev/null +++ b/astrobwt/astrobwt_fast/endian_little.go @@ -0,0 +1,9 @@ +//go:build amd64 || amd64p32 || 386 || arm || arm64 || mipsle || mips64le || mips64p32le || ppc64le || riscv || riscv64 || wasm || loong64 +// +build amd64 amd64p32 386 arm arm64 mipsle mips64le mips64p32le ppc64le riscv riscv64 wasm loong64 + +package astrobwt_fast + +const LittleEndian = true +const BigEndian = false + +//see https://github.com/golang/go/blob/master/src/go/build/syslist.go diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index dcb7c103..ae5cfb16 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -209,7 +209,7 @@ func Blockchain_Start(params map[string]interface{}) (*Blockchain, error) { logger.Info("Chain Pruned till", "topoheight", chain.Pruned) } - // detect case if chain was corrupted earlier,so as it can be deleted and resynced + // detect case if chain was corrupted earlier,so as it can be deleted and resynced if chain.Pruned < globals.Config.HF1_HEIGHT && globals.IsMainnet() && chain.Get_Height() >= globals.Config.HF1_HEIGHT+1 { toporecord, err := chain.Store.Topo_store.Read(globals.Config.HF1_HEIGHT + 1) if err != nil { diff --git a/build_all.sh b/build_all.sh index e8beb558..7933b82f 100644 --- a/build_all.sh +++ b/build_all.sh @@ -16,7 +16,7 @@ bash $ABSDIR/build_package.sh "./cmd/derod" bash $ABSDIR/build_package.sh "./cmd/explorer" bash $ABSDIR/build_package.sh "./cmd/dero-wallet-cli" bash $ABSDIR/build_package.sh "./cmd/dero-miner" -#bash $ABSDIR/build_package.sh "./cmd/simulator" +bash $ABSDIR/build_package.sh "./cmd/simulator" #bash $ABSDIR/build_package.sh "./cmd/rpc_examples/pong_server" @@ -36,8 +36,8 @@ go run github.com/randall77/makefat ./dero_darwin_universal/derod-darwin ./dero go run github.com/randall77/makefat ./dero_darwin_universal/explorer-darwin ./dero_darwin_amd64/explorer-darwin-amd64 ./dero_darwin_arm64/explorer-darwin-arm64 go run github.com/randall77/makefat ./dero_darwin_universal/dero-wallet-cli-darwin ./dero_darwin_amd64/dero-wallet-cli-darwin-amd64 ./dero_darwin_arm64/dero-wallet-cli-darwin-arm64 go run github.com/randall77/makefat ./dero_darwin_universal/dero-miner-darwin ./dero_darwin_amd64/dero-miner-darwin-amd64 ./dero_darwin_arm64/dero-miner-darwin-arm64 -#go run github.com/randall77/makefat ./dero_darwin_universal/simulator-darwin ./dero_darwin_amd64/simulator-darwin-amd64 ./dero_darwin_arm64/simulator-darwin-arm64 -go run github.com/randall77/makefat ./dero_darwin_universal/pong_server-darwin ./dero_darwin_amd64/pong_server-darwin-amd64 ./dero_darwin_arm64/pong_server-darwin-arm64 +go run github.com/randall77/makefat ./dero_darwin_universal/simulator-darwin ./dero_darwin_amd64/simulator-darwin-amd64 ./dero_darwin_arm64/simulator-darwin-arm64 +#go run github.com/randall77/makefat ./dero_darwin_universal/pong_server-darwin ./dero_darwin_amd64/pong_server-darwin-amd64 ./dero_darwin_arm64/pong_server-darwin-arm64 rm -rf dero_darwin_amd64 rm -rf dero_darwin_arm64 diff --git a/cmd/dero-miner/miner.go b/cmd/dero-miner/miner.go index 9d1d7135..f44648c5 100644 --- a/cmd/dero-miner/miner.go +++ b/cmd/dero-miner/miner.go @@ -46,7 +46,7 @@ import "github.com/deroproject/derohe/rpc" import "github.com/chzyer/readline" import "github.com/docopt/docopt-go" -import "github.com/deroproject/derohe/pow" +import "github.com/deroproject/derohe/astrobwt/astrobwt_fast" import "github.com/gorilla/websocket" @@ -75,8 +75,8 @@ ONE CPU, ONE VOTE. http://wiki.dero.io Usage: - dero-miner --wallet-address= [--daemon-rpc-address=<127.0.0.1:10102>] [--mining-threads=] [--testnet] [--debug] - dero-miner --bench [--max-pow-size=1120] + dero-miner --wallet-address= [--daemon-rpc-address=] [--mining-threads=] [--testnet] [--debug] + dero-miner --bench dero-miner -h | --help dero-miner --version @@ -84,12 +84,12 @@ Options: -h --help Show this screen. --version Show version. --bench Run benchmark mode. - --daemon-rpc-address=<127.0.0.1:10102> Miner will connect to daemon RPC on this port. + --daemon-rpc-address=<127.0.0.1:10102> Miner will connect to daemon RPC on this port (default minernode1.dero.live:10100). --wallet-address= This address is rewarded when a block is mined sucessfully. --mining-threads= Number of CPU threads for mining [default: ` + fmt.Sprintf("%d", runtime.GOMAXPROCS(0)) + `] -Example Mainnet: ./dero-miner-linux-amd64 --wallet-address dero1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqhmy4zf --daemon-rpc-address=http://explorer.dero.io:10102 -Example Testnet: ./dero-miner-linux-amd64 --wallet-address deto1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqdzcd8p --daemon-rpc-address=http://127.0.0.1:40402 +Example Mainnet: ./dero-miner-linux-amd64 --wallet-address dero1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqhmy4zf --daemon-rpc-address=minernode1.dero.live:10100 +Example Testnet: ./dero-miner-linux-amd64 --wallet-address deto1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqdzcd8p --daemon-rpc-address=127.0.0.1:40402 If daemon running on local machine no requirement of '--daemon-rpc-address' argument. ` var Exit_In_Progress = make(chan bool) @@ -154,7 +154,7 @@ func main() { } if !globals.Arguments["--testnet"].(bool) { - daemon_rpc_address = "127.0.0.1:10100" + daemon_rpc_address = "minernode1.dero.live:10100" } else { daemon_rpc_address = "127.0.0.1:10100" } @@ -379,10 +379,11 @@ func random_execution(wg *sync.WaitGroup, iterations int) { runtime.LockOSThread() //threadaffinity() + scratch := astrobwt_fast.Pool.Get().(*astrobwt_fast.ScratchData) rand.Read(workbuf[:]) for i := 0; i < iterations; i++ { - _ = pow.Pow(workbuf[:]) + _ = astrobwt_fast.POW_optimized(workbuf[:], scratch) } wg.Done() runtime.UnlockOSThread() @@ -446,7 +447,9 @@ func mineblock(tid int) { var diff big.Int var work [block.MINIBLOCK_SIZE]byte - time.Sleep(5*time.Second) + scratch := astrobwt_fast.Pool.Get().(*astrobwt_fast.ScratchData) + + time.Sleep(5 * time.Second) nonce_buf := work[block.MINIBLOCK_SIZE-5:] //since slices are linked, it modifies parent runtime.LockOSThread() @@ -483,7 +486,7 @@ func mineblock(tid int) { i++ binary.BigEndian.PutUint32(nonce_buf, i) - powhash := pow.Pow(work[:]) + powhash := astrobwt_fast.POW_optimized(work[:], scratch) atomic.AddUint64(&counter, 1) if CheckPowHashBig(powhash, &diff) == true { diff --git a/config/version.go b/config/version.go index 86af828a..2b11084c 100644 --- a/config/version.go +++ b/config/version.go @@ -20,4 +20,4 @@ import "github.com/blang/semver/v4" // right now it has to be manually changed // do we need to include git commitsha?? -var Version = semver.MustParse("3.4.133-48.DEROHE.STARGATE+26022022") +var Version = semver.MustParse("3.4.138-49.DEROHE.STARGATE+10032022") diff --git a/walletapi/wallet_disk.go b/walletapi/wallet_disk.go index 80c3f364..e8b059ea 100644 --- a/walletapi/wallet_disk.go +++ b/walletapi/wallet_disk.go @@ -138,6 +138,10 @@ func (w *Wallet_Disk) Save_Wallet() (err error) { return } + // attempt to protect atleast one backup copy + os.Remove(w.filename + ".bak") + os.Rename(w.filename, w.filename+".bak") // renames are atomic, thus more chances of retention + return ioutil.WriteFile(w.filename, w.Wallet_Memory.db_memory, 0600) } diff --git a/walletapi/wallet_transfer.go b/walletapi/wallet_transfer.go index 161b1644..e81125ab 100644 --- a/walletapi/wallet_transfer.go +++ b/walletapi/wallet_transfer.go @@ -203,6 +203,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint // try to resolve name to address here if _, err = rpc.NewAddress(transfers[t].Destination); err != nil { if transfers[t].Destination, err = w.NameToAddress(transfers[t].Destination); err != nil { + err = fmt.Errorf("could not decode name or address err '%s' name '%s'\n", err, transfers[t].Destination) return } } @@ -225,6 +226,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint // we currently bypass this since random members are chosen which have not been used in last 5 block _, noncetopo, block_hash, self_e, err := w.GetEncryptedBalanceAtTopoHeight(zeroscid, -1, w.GetAddress().String()) if err != nil { + err = fmt.Errorf("could not obtain encrypted balance for self err %s\n", err) return } @@ -242,6 +244,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint er, err := w.GetSelfEncryptedBalanceAtTopoHeight(transfers[0].SCID, topoheight) if err != nil { + err = fmt.Errorf("could not obtain encrypted balance for self err %s\n", err) return } height := uint64(er.Height)