Reproducible builds implemented. Pruning & fastsync fixes.

astrobwt/suffixarray.go

@@ -13,7 +13,6 @@
 //	// lookup byte slice s
 //	offsets1 := index.Lookup(s, -1) // the list of all indices where s occurs in data
 //	offsets2 := index.Lookup(s, 3)  // the list of at most 3 indices where s occurs in data
-//
 package astrobwt
 
 import (
@@ -230,7 +229,6 @@ func (x *Index) Write(w io.Writer) error {
 
 // Bytes returns the data over which the index was created.
 // It must not be modified.
-//
 func (x *Index) Bytes() []byte {
 	return x.data
 }
@@ -255,7 +253,6 @@ func (x *Index) lookupAll(s []byte) ints {
 // The result is nil if s is empty, s is not found, or n == 0.
 // Lookup time is O(log(N)*len(s) + len(result)) where N is the
 // size of the indexed data.
-//
 func (x *Index) Lookup(s []byte, n int) (result []int) {
 	if len(s) > 0 && n != 0 {
 		matches := x.lookupAll(s)
@@ -286,7 +283,6 @@ func (x *Index) Lookup(s []byte, n int) (result []int) {
 // in successive order. Otherwise, at most n matches are returned and
 // they may not be successive. The result is nil if there are no matches,
 // or if n == 0.
-//
 func (x *Index) FindAllIndex(r *regexp.Regexp, n int) (result [][]int) {
 	// a non-empty literal prefix is used to determine possible
 	// match start indices with Lookup

block/block.go

@@ -185,7 +185,7 @@ func (bl *Block) GetTXSHash() (result crypto.Hash) {
 	return
 }
 
-//parse entire block completely
+// parse entire block completely
 func (bl *Block) Deserialize(buf []byte) (err error) {
 	done := 0
 

block/miniblock.go

@@ -172,7 +172,7 @@ func (mbl *MiniBlock) Serialize() (result []byte) {
 	return b.Bytes()
 }
 
-//parse entire block completely
+// parse entire block completely
 func (mbl *MiniBlock) Deserialize(buf []byte) (err error) {
 	if len(buf) < MINIBLOCK_SIZE {
 		return fmt.Errorf("Expected %d bytes. Actual %d", MINIBLOCK_SIZE, len(buf))

blockchain/miner_block.go

@@ -50,7 +50,9 @@ type BlockScore struct {
 }
 
 // Heighest height is ordered first,  the condition is reverted see eg. at https://golang.org/pkg/sort/#Slice
-//  if heights are equal, nodes are sorted by their block ids which will never collide , hopefullly
+//
+//	if heights are equal, nodes are sorted by their block ids which will never collide , hopefullly
+//
 // block ids are sorted by lowest byte first diff
 func sort_descending_by_height_blid(tips_scores []BlockScore) {
 	sort.Slice(tips_scores, func(i, j int) bool {
@@ -101,7 +103,7 @@ func convert_uint32_to_crypto_hash(i uint32) crypto.Hash {
 	return h
 }
 
-//NOTE: this function is quite big since we do a lot of things in preparation of next blocks
+// NOTE: this function is quite big since we do a lot of things in preparation of next blocks
 func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address) (cbl *block.Complete_Block, bl block.Block, err error) {
 	//chain.Lock()
 	//defer chain.Unlock()
@@ -343,7 +345,6 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address) (cbl
 	return
 }
 
-//
 func ConvertBlockToMiniblock(bl block.Block, miniblock_miner_address rpc.Address) (mbl block.MiniBlock) {
 	mbl.Version = 1
 
@@ -480,7 +481,7 @@ func (chain *Blockchain) Accept_new_block(tstamp uint64, miniblock_blob []byte)
 	// lets try to check pow to detect whether the miner is cheating
 
 	if !chain.simulator && !chain.VerifyMiniblockPoW(&bl, mbl) {
-		logger.V(1).Error(err, "Error ErrInvalidPoW")
+		//logger.V(1).Error(err, "Error ErrInvalidPoW")
 		err = errormsg.ErrInvalidPoW
 		return
 	}

blockchain/prune_history.go

@@ -117,9 +117,14 @@ func Prune_Blockchain(prune_topo int64) (err error) {
 // any error while deleting should be considered non fatal
 func discard_blocks_and_transactions(store *storage, topoheight int64) {
 
-	globals.Logger.Info("Block store before pruning", "size", ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store"))))
+	size_before := ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store")))
 
 	for i := int64(0); i < topoheight-20; i++ { // donot some more blocks for sanity currently
+
+		if i%1000 == 0 {
+			globals.Logger.Info("Deleting old block/txs", "done", float64(i*100)/float64(topoheight-20))
+		}
+
 		if toporecord, err := store.Topo_store.Read(i); err == nil {
 			blid := toporecord.BLOCK_ID
 
@@ -138,6 +143,7 @@ func discard_blocks_and_transactions(store *storage, topoheight int64) {
 			}
 		}
 	}
+	globals.Logger.Info("Block store before pruning", "size", size_before)
 	globals.Logger.Info("Block store after pruning ", "size", ByteCountIEC(DirSize(filepath.Join(store.Block_tx_store.basedir, "bltx_store"))))
 }
 
@@ -180,6 +186,12 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates
 		}
 	}
 
+	if latest_commit_version, err = graviton.Commit(write_balance_tree); err != nil {
+		return 0, err
+	}
+
+	globals.Logger.Info("Main balance tree cloned")
+
 	/*	h,_ := old_balance_tree.Hash()
 		fmt.Printf("old balance hash %+v\n",h )
 		h,_ = write_balance_tree.Hash()
@@ -214,14 +226,18 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates
 		}
 	}
 
+	if latest_commit_version, err = graviton.Commit(write_meta_tree); err != nil {
+		return 0, err
+	}
+
 	/*	h,_ = old_meta_tree.Hash()
 		fmt.Printf("old meta hash %+v\n",h )
 		h,_ = write_meta_tree.Hash()
 		fmt.Printf("new meta hash %+v\n",h )
 
 		os.Exit(0)
 	*/
-	var sc_trees []*graviton.Tree
+	sc_names := map[string]bool{}
 	// now we have to copy all scs data one by one
 	for _, scid := range sc_list {
 		var old_sc_tree, write_sc_tree *graviton.Tree
@@ -235,12 +251,13 @@ func clone_snapshot(rsource, wsource *graviton.Store, r_ssversion uint64) (lates
 		for k, v, err := c.First(); err == nil; k, v, err = c.Next() {
 			write_sc_tree.Put(k, v)
 		}
-		sc_trees = append(sc_trees, write_sc_tree)
+		sc_names[string(scid)] = true
+		if latest_commit_version, err = graviton.Commit(write_sc_tree); err != nil {
+			return
+		}
 	}
+	globals.Logger.Info("SCs cloned")
 
-	sc_trees = append(sc_trees, write_balance_tree)
-	sc_trees = append(sc_trees, write_meta_tree)
-	latest_commit_version, err = graviton.Commit(sc_trees...)
 	return
 }
 
@@ -309,7 +326,6 @@ func diff_snapshot(rsource, wsource *graviton.Store, old_version uint64, new_ver
 
 	// now we have to copy new scs data one by one
 	for _, scid := range sc_list_new {
-
 		if old_tree, err = old_ss.GetTree(string(scid)); err != nil {
 			return
 		}
@@ -319,7 +335,7 @@ func diff_snapshot(rsource, wsource *graviton.Store, old_version uint64, new_ver
 		if write_tree, err = write_ss.GetTree(string(scid)); err != nil {
 			return
 		}
-		c := old_tree.Cursor()
+		c := new_tree.Cursor()
 		for k, v, err := c.First(); err == nil; k, v, err = c.Next() {
 			write_tree.Put(k, v)
 		}
@@ -399,6 +415,13 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig
 
 				}
 			}
+			if err != nil {
+				return
+			}
+
+			if i%1000 == 0 {
+				globals.Logger.Info("Commiting block to block changes", "done", float64(i*100)/float64(max_topoheight))
+			}
 		}
 
 		// now lets store all the commit versions in 1 go
@@ -444,6 +467,10 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig
 				return err
 			}
 
+			if i%1000 == 0 {
+				globals.Logger.Info("Rewriting entries", "done", float64(i*100)/float64(len(new_entries)))
+			}
+
 		}
 	}
 
@@ -456,6 +483,9 @@ func rewrite_graviton_store(store *storage, prune_topoheight int64, max_topoheig
 			globals.Logger.Error(err, "err reading toporecord", "topo", i)
 			return err // this is irrepairable  damage
 		}
+		if i%1000 == 0 {
+			globals.Logger.Info("Filling gaps", "done", float64(i*100)/float64(prune_topoheight))
+		}
 	}
 
 	// now lets remove the old graviton db
@@ -484,5 +514,9 @@ func clone_tree_changes(old_tree, new_tree, write_tree *graviton.Tree) {
 		write_tree.Put(k, new_value)
 	}
 
-	graviton.Diff(old_tree, new_tree, nil, modify_handler, insert_handler)
+	delete_handler := func(k, v []byte) { // modification receives old value
+		write_tree.Delete(k)
+	}
+
+	graviton.Diff(old_tree, new_tree, delete_handler, modify_handler, insert_handler)
 }

blockchain/storefs.go

@@ -341,6 +341,8 @@ func (s *storefs) migrate_old_tx() {
 		defer func() {
 			fmt.Printf("Migrated %d txs to new structure\n", migrated)
 		}()
+	} else {
+		return
 	}
 
 	if err != nil {
@@ -365,8 +367,8 @@ func (s *storefs) migrate_old_tx() {
 			}
 
 			copy(h[:], txhash[:])
-			s.WriteTX(h, data)
 			s.DeleteTX(h) // this will delete legacy version
+			s.WriteTX(h, data)
 			migrated++
 
 		}

blockchain/storetopo.go

@@ -35,7 +35,8 @@ const TOPORECORD_SIZE int64 = 48
 
 // this file implements a filesystem store which is used to store topo to block mapping directly in the file system and the state version directly tied
 type storetopofs struct {
-	topomapping *os.File
+	topomapping        *os.File
+	last_state_version uint64
 }
 
 func (s TopoRecord) String() string {
@@ -82,6 +83,7 @@ func (s *storetopofs) Read(index int64) (TopoRecord, error) {
 func (s *storetopofs) Write(index int64, blid [32]byte, state_version uint64, height int64) (err error) {
 	var buf [TOPORECORD_SIZE]byte
 	var record TopoRecord
+	var zero_hash [32]byte
 
 	copy(buf[:], blid[:])
 	binary.LittleEndian.PutUint64(buf[len(record.BLOCK_ID):], state_version)
@@ -90,11 +92,18 @@ func (s *storetopofs) Write(index int64, blid [32]byte, state_version uint64, he
 	binary.LittleEndian.PutUint64(buf[len(record.BLOCK_ID)+8:], uint64(height))
 
 	_, err = s.topomapping.WriteAt(buf[:], index*TOPORECORD_SIZE)
-
-	s.topomapping.Sync() // looks like this is the cause of corruption
+	if s.last_state_version != state_version || state_version == 0 {
+		if blid != zero_hash { // during fast sync avoid syncing overhead
+			s.topomapping.Sync() // looks like this is the cause of corruption
+		}
+	}
+	s.last_state_version = state_version
 
 	return err
 }
+func (s *storetopofs) Sync() {
+	s.topomapping.Sync()
+}
 
 func (s *storetopofs) Clean(index int64) (err error) {
 	var state_version uint64
@@ -163,6 +172,10 @@ func (s *storetopofs) LocatePruneTopo() int64 {
 
 	prune_topo--
 
+	if prune_topo > count {
+		panic("invalid prune detected")
+	}
+
 	pruned_till = prune_topo
 	return prune_topo
 }

cmd/derod/rpc/websocket_getwork_server.go

@@ -73,7 +73,7 @@ var miners_count int
 var mini_found_time []int64 // this array contains a epoch timestamp in int64
 var rate_lock sync.Mutex
 
-//this function will return wrong result if too wide time glitches happen to system clock
+// this function will return wrong result if too wide time glitches happen to system clock
 func Counter(seconds int64) (r int) { // we need atleast 1 mini to find a rate
 	rate_lock.Lock()
 	defer rate_lock.Unlock()

cmd/derod/rpc/websocket_server.go

@@ -58,7 +58,7 @@ type RPCServer struct {
 	sync.RWMutex
 }
 
-//var Exit_In_Progress bool
+// var Exit_In_Progress bool
 var chain *blockchain.Blockchain
 var logger logr.Logger
 
@@ -296,7 +296,6 @@ func Echo(ctx context.Context, args []string) string {
 var internal_server = server.NewLocal(historical_apis, nil) // uses traditional "getinfo" for compatibility reasons
 // Bridge HTTP to the JSON-RPC server.
 var bridge = jhttp.NewBridge(internal_server.Client)
-
 */
 var historical_apis = handler.Map{"getinfo": handler.New(GetInfo),
 	"get_info":                   handler.New(GetInfo), // this is just an alias to above

config/seed_nodes.go

@@ -22,9 +22,9 @@ package config
 // only version 2
 var Mainnet_seed_nodes = []string{
 	"89.38.99.117:8443",
-	"45.82.66.54:8080",
+	"109.236.81.137:8080",
 	"89.38.97.110:11011",
-	"45.82.66.55:11011",
+	"190.2.136.120:11011",
 }
 
 // some seed node for testnet

config/version.go

@@ -20,4 +20,4 @@ import "github.com/blang/semver/v4"
 
 // right now it has to be manually changed
 // do we need to include git commitsha??
-var Version = semver.MustParse("3.5.2-113.DEROHE.STARGATE+01102022")
+var Version = semver.MustParse("3.5.2-114.DEROHE.STARGATE+01102022")

cryptography/crypto/algebra_fieldvector.go

@@ -55,7 +55,7 @@ func (fv *FieldVector) Slice(start, end int) *FieldVector {
 	return &result
 }
 
-//copy and return
+// copy and return
 func (fv *FieldVector) Clone() *FieldVector {
 	return fv.Slice(0, len(fv.vector))
 }