From 50b376e921ddc49a1ec5e52189516df2ed560531 Mon Sep 17 00:00:00 2001 From: Captain Date: Wed, 28 Sep 2022 16:51:11 +0000 Subject: [PATCH] Optimizing Inefficient Functions Usecase. --- blockchain/store.go | 3 +- blockchain/storefs.go | 256 ++++++++++++++++++++++++++++++++--------- config/version.go | 2 +- p2p/connection_pool.go | 1 + 4 files changed, 203 insertions(+), 59 deletions(-) mode change 100644 => 100755 blockchain/storefs.go diff --git a/blockchain/store.go b/blockchain/store.go index 3d295b30..26969556 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -42,6 +42,7 @@ func (s *storage) Initialize(params map[string]interface{}) (err error) { if s.Balance_store, err = graviton.NewDiskStore(filepath.Join(current_path, "balances")); err == nil { if err = s.Topo_store.Open(current_path); err == nil { s.Block_tx_store.basedir = current_path + s.Block_tx_store.migrate_old_tx() } } @@ -292,7 +293,7 @@ func (chain *Blockchain) Load_Block_Topological_order_at_index(index_pos int64) } -//load store hash from 2 tree +// load store hash from 2 tree func (chain *Blockchain) Load_Merkle_Hash(version uint64) (hash crypto.Hash, err error) { if hashi, ok := chain.cache_VersionMerkle.Get(version); ok { diff --git a/blockchain/storefs.go b/blockchain/storefs.go old mode 100644 new mode 100755 index 3b437e8d..701372df --- a/blockchain/storefs.go +++ b/blockchain/storefs.go @@ -18,12 +18,14 @@ package blockchain // this file implements a filesystem store which is used to store blocks/transactions directly in the file system +import "io" import "os" import "fmt" import "strings" import "io/ioutil" import "math/big" import "path/filepath" +import "encoding/hex" import "github.com/deroproject/derohe/globals" type storefs struct { @@ -38,6 +40,13 @@ func (s *storefs) getpath(h [32]byte) string { return filepath.Join(filepath.Join(s.basedir, "bltx_store"), fmt.Sprintf("%02x", h[0]), fmt.Sprintf("%02x", h[1])) } +func (s *storefs) getpathtx(h [32]byte) string { + // if you wish to use 3 level indirection, it will cause 16 million inodes to be used, but system will be faster + //return filepath.Join(filepath.Join(s.basedir, "bltx_store"), fmt.Sprintf("%02x", h[0]), fmt.Sprintf("%02x", h[1]), fmt.Sprintf("%02x", h[2])) + // currently we are settling on 65536 inodes + return filepath.Join(filepath.Join(s.basedir, "bltx_store"), fmt.Sprintf("%02x", h[16]), fmt.Sprintf("%02x", h[17])) +} + // the filename stores the following information // hex block id (64 chars).block._ rewards (decimal) _ difficulty _ cumulative difficulty @@ -50,17 +59,28 @@ func (s *storefs) ReadBlock(h [32]byte) ([]byte, error) { dir := s.getpath(h) - files, err := os.ReadDir(dir) + fd, err := os.Open(dir) if err != nil { return nil, err } + defer fd.Close() filename_start := fmt.Sprintf("%x.block", h[:]) - for _, file := range files { - if strings.HasPrefix(file.Name(), filename_start) { - //fmt.Printf("Reading block with filename %s\n", file.Name()) - file := filepath.Join(dir, file.Name()) - return os.ReadFile(file) + + for { + files, err := fd.Readdirnames(1024) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + for _, file := range files { + if strings.HasPrefix(file, filename_start) { + //fmt.Printf("Reading block with filename %s\n", file.Name()) + file := filepath.Join(dir, file) + return os.ReadFile(file) + } } } @@ -71,23 +91,35 @@ func (s *storefs) ReadBlock(h [32]byte) ([]byte, error) { func (s *storefs) DeleteBlock(h [32]byte) error { dir := s.getpath(h) - files, err := os.ReadDir(dir) + fd, err := os.Open(dir) if err != nil { return err } + defer fd.Close() filename_start := fmt.Sprintf("%x.block", h[:]) var found bool - for _, file := range files { - if strings.HasPrefix(file.Name(), filename_start) { - file := filepath.Join(dir, file.Name()) - err = os.Remove(file) - if err != nil { - //return err + + for { + files, err := fd.Readdirnames(1024) + if err == io.EOF { + break + } + if err != nil { + return err + } + for _, file := range files { + if strings.HasPrefix(file, filename_start) { + file := filepath.Join(dir, file) + err = os.Remove(file) + if err != nil { + //return err + } + found = true } - found = true } } + if found { return nil } @@ -98,27 +130,37 @@ func (s *storefs) DeleteBlock(h [32]byte) error { func (s *storefs) ReadBlockDifficulty(h [32]byte) (*big.Int, error) { dir := s.getpath(h) - files, err := os.ReadDir(dir) + fd, err := os.Open(dir) if err != nil { return nil, err } + defer fd.Close() filename_start := fmt.Sprintf("%x.block", h[:]) - for _, file := range files { - if strings.HasPrefix(file.Name(), filename_start) { + for { + files, err := fd.Readdirnames(1024) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } - diff := new(big.Int) + for _, file := range files { + if strings.HasPrefix(file, filename_start) { + diff := new(big.Int) - parts := strings.Split(file.Name(), "_") - if len(parts) != 4 { - panic("such filename cannot occur") - } + parts := strings.Split(file, "_") + if len(parts) != 4 { + panic("such filename cannot occur") + } - _, err := fmt.Sscan(parts[1], diff) - if err != nil { - return nil, err + _, err := fmt.Sscan(parts[1], diff) + if err != nil { + return nil, err + } + return diff, nil } - return diff, nil } } @@ -132,33 +174,50 @@ func (chain *Blockchain) ReadBlockSnapshotVersion(h [32]byte) (uint64, error) { func (s *storefs) ReadBlockSnapshotVersion(h [32]byte) (uint64, error) { dir := s.getpath(h) - files, err := os.ReadDir(dir) // this always returns the sorted list + fd, err := os.Open(dir) if err != nil { return 0, err } + defer fd.Close() + // windows has a caching issue, so earlier versions may exist at the same time - // so we mitigate it, by using the last version, below 3 lines reverse the already sorted arrray - for left, right := 0, len(files)-1; left < right; left, right = left+1, right-1 { - files[left], files[right] = files[right], files[left] - } + // so we mitigate it, by using the highest version filename_start := fmt.Sprintf("%x.block", h[:]) - for _, file := range files { - if strings.HasPrefix(file.Name(), filename_start) { - var ssversion uint64 - parts := strings.Split(file.Name(), "_") - if len(parts) != 4 { - panic("such filename cannot occur") - } - _, err := fmt.Sscan(parts[2], &ssversion) - if err != nil { - return 0, err + + var biggest_ss_version uint64 + for { + files, err := fd.Readdirnames(1024) + if err == io.EOF { + break + } + if err != nil { + return 0, err + } + for _, file := range files { + if strings.HasPrefix(file, filename_start) { + var ssversion uint64 + parts := strings.Split(file, "_") + if len(parts) != 4 { + panic("such filename cannot occur") + } + _, err := fmt.Sscan(parts[2], &ssversion) + if err != nil { + return 0, err + } + if ssversion > biggest_ss_version { + biggest_ss_version = ssversion + } + } - return ssversion, nil } } + if biggest_ss_version > 0 { + return biggest_ss_version, nil + } return 0, os.ErrNotExist + } func (chain *Blockchain) ReadBlockHeight(h [32]byte) (uint64, error) { @@ -177,24 +236,35 @@ func (chain *Blockchain) ReadBlockHeight(h [32]byte) (uint64, error) { func (s *storefs) ReadBlockHeight(h [32]byte) (uint64, error) { dir := s.getpath(h) - files, err := os.ReadDir(dir) + fd, err := os.Open(dir) if err != nil { return 0, err } + defer fd.Close() filename_start := fmt.Sprintf("%x.block", h[:]) - for _, file := range files { - if strings.HasPrefix(file.Name(), filename_start) { - var height uint64 - parts := strings.Split(file.Name(), "_") - if len(parts) != 4 { - panic("such filename cannot occur") - } - _, err := fmt.Sscan(parts[3], &height) - if err != nil { - return 0, err + for { + files, err := fd.Readdirnames(1024) + if err == io.EOF { + break + } + if err != nil { + return 0, err + } + + for _, file := range files { + if strings.HasPrefix(file, filename_start) { + var height uint64 + parts := strings.Split(file, "_") + if len(parts) != 4 { + panic("such filename cannot occur") + } + _, err := fmt.Sscan(parts[3], &height) + if err != nil { + return 0, err + } + return height, nil } - return height, nil } } @@ -211,13 +281,21 @@ func (s *storefs) WriteBlock(h [32]byte, data []byte, difficulty *big.Int, ss_ve } func (s *storefs) ReadTX(h [32]byte) ([]byte, error) { - dir := s.getpath(h) + { // legacy code + dir := s.getpath(h) + file := filepath.Join(dir, fmt.Sprintf("%x.tx", h[:])) + if data, err := ioutil.ReadFile(file); err == nil { + return data, err + } + } + + dir := s.getpathtx(h) file := filepath.Join(dir, fmt.Sprintf("%x.tx", h[:])) return ioutil.ReadFile(file) } func (s *storefs) WriteTX(h [32]byte, data []byte) (err error) { - dir := s.getpath(h) + dir := s.getpathtx(h) file := filepath.Join(dir, fmt.Sprintf("%x.tx", h[:])) if err = os.MkdirAll(dir, 0700); err != nil { @@ -228,7 +306,71 @@ func (s *storefs) WriteTX(h [32]byte, data []byte) (err error) { } func (s *storefs) DeleteTX(h [32]byte) (err error) { - dir := s.getpath(h) + + { // legacy code + dir := s.getpath(h) + file := filepath.Join(dir, fmt.Sprintf("%x.tx", h[:])) + err = os.Remove(file) + if err == nil { + return + } + } + + dir := s.getpathtx(h) file := filepath.Join(dir, fmt.Sprintf("%x.tx", h[:])) return os.Remove(file) } + +// migrate old tx folder structure to new structure +func (s *storefs) migrate_old_tx() { + var h [32]byte + dir := s.getpath(h) + + fd, err := os.Open(dir) + if err != nil { + return + } + defer fd.Close() + + migrated := 0 + + files, err := fd.Readdirnames(0) + + if len(files) > 99*1024 { + fmt.Printf("Migrating old tx data, Please wait, it might take couple of secs to minutes (depending on storage speed).\n") + defer func() { + fmt.Printf("Migrated %d txs to new structure\n", migrated) + }() + } + + if err != nil { + return + } + + for _, file := range files { + if strings.HasSuffix(file, ".tx") { + + data, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + continue + } + + txhash, err := hex.DecodeString(strings.TrimSuffix(file, ".tx")) + if err != nil { + continue + } + + if migrated%1000 == 0 { + fmt.Printf("migrated %d/%d files\n", migrated, len(files)) + } + + copy(h[:], txhash[:]) + s.WriteTX(h, data) + s.DeleteTX(h) // this will delete legacy version + migrated++ + + } + + } + +} diff --git a/config/version.go b/config/version.go index fcf9dd10..5cc3f7cb 100644 --- a/config/version.go +++ b/config/version.go @@ -20,4 +20,4 @@ import "github.com/blang/semver/v4" // right now it has to be manually changed // do we need to include git commitsha?? -var Version = semver.MustParse("3.5.1-96.DEROHE.STARGATE+26022022") +var Version = semver.MustParse("3.5.1-110a.DEROHE.STARGATE+26022022") diff --git a/p2p/connection_pool.go b/p2p/connection_pool.go index 0b5189fd..a844522e 100644 --- a/p2p/connection_pool.go +++ b/p2p/connection_pool.go @@ -313,6 +313,7 @@ func Connection_Print() { fmt.Print(color_normal) } + logger.Info("Connection info for peers", "count", len(clist), "our Statehash", StateHash) }