DERO-HE STARGATE Mainnet Release49

astrobwt/astrobwt_fast/astrobwt_optimized.go

@@ -0,0 +1,162 @@
+package astrobwt_fast
+
+import "unsafe"
+import "hash"
+import "sync"
+import "crypto/rand"
+import "encoding/binary"
+import "golang.org/x/crypto/sha3"
+import "golang.org/x/crypto/salsa20/salsa"
+
+const stage1_length uint32 = 9973 // it is a prime
+
+// see here to improve the algorithms more https://github.com/y-256/libdivsufsort/blob/wiki/SACA_Benchmarks.md
+// this optimized algorithm is used only  in the miner and not in the blockchain
+
+type ScratchData struct {
+	hasher              hash.Hash
+	stage1              [stage1_length + 64]byte // 10 KB stages are taken from it
+	stage1_result       *[stage1_length + 1]uint16
+	stage1_result_bytes *[(stage1_length) * 2]uint8
+	indices             [stage1_length + 1]uint32 // 40 KB
+	tmp_indices         [stage1_length + 1]uint32 // 40 KB
+}
+
+var Pool = sync.Pool{New: func() interface{} {
+	var d ScratchData
+	d.hasher = sha3.New256()
+	d.stage1_result = ((*[stage1_length + 1]uint16)(unsafe.Pointer(&d.indices[0])))
+	d.stage1_result_bytes = ((*[(stage1_length) * 2]byte)(unsafe.Pointer(&d.indices[0])))
+
+	return &d
+}}
+
+func POW_optimized(inputdata []byte, data *ScratchData) (outputhash [32]byte) {
+
+	defer func() {
+		if r := recover(); r != nil { // if something happens due to RAM issues in miner, we should continue, system will crash sooner or later
+			var buf [16]byte
+			rand.Read(buf[:])
+			outputhash = sha3.Sum256(buf[:]) // return a falsified has which will fail the check
+		}
+	}()
+
+	var key [32]byte
+	for i := range data.stage1 {
+		data.stage1[i] = 0
+	}
+
+	var counter [16]byte
+
+	data.hasher.Reset()
+	data.hasher.Write(inputdata)
+	_ = data.hasher.Sum(key[:0])
+
+	salsa.XORKeyStream(data.stage1[:stage1_length], data.stage1[:stage1_length], &counter, &key)
+	sort_indices(stage1_length, data.stage1[:stage1_length+40], data.stage1_result[:], data) // extra 40 bytes since we may read them, but we never write them
+
+	if LittleEndian {
+		data.hasher.Reset()
+		data.hasher.Write(data.stage1_result_bytes[:])
+		_ = data.hasher.Sum(key[:0])
+	} else {
+		var s [stage1_length * 2]byte
+		for i, c := range data.stage1_result {
+			binary.LittleEndian.PutUint16(s[i<<1:], c)
+		}
+		data.hasher.Reset()
+		data.hasher.Write(s[:])
+		_ = data.hasher.Sum(key[:0])
+	}
+
+	copy(outputhash[:], key[:])
+	return
+}
+
+func fix(v []byte, indices []uint32, i int) {
+	prev_t := indices[i]
+	t := indices[i+1]
+
+	data_a := binary.BigEndian.Uint32(v[((t)&0xffff)+2:])
+	if data_a < binary.BigEndian.Uint32(v[((prev_t)&0xffff)+2:]) {
+		t2 := prev_t
+		j := i
+		_ = indices[j+1]
+		for {
+			indices[j+1] = prev_t
+			j--
+			if j < 0 {
+				break
+			}
+			prev_t = indices[j]
+			if (t^prev_t) <= 0xffff && data_a < binary.BigEndian.Uint32(v[((prev_t)&0xffff)+2:]) {
+				continue
+			} else {
+				break
+			}
+		}
+		indices[j+1] = t
+		t = t2
+	}
+}
+
+// basically
+func sort_indices(N uint32, v []byte, output []uint16, d *ScratchData) {
+
+	var byte_counters [2][256]byte
+	var counters [2][256]uint16
+
+	v[N] = 0 // make sure extra byte accessed is zero
+
+	indices := d.indices[:]
+	tmp_indices := d.tmp_indices[:]
+
+	for _, c := range v[:N] {
+		byte_counters[1][c]++
+	}
+	byte_counters[0] = byte_counters[1]
+	byte_counters[0][v[0]]--
+
+	counters[0][0] = uint16(byte_counters[0][0])
+	counters[1][0] = uint16(byte_counters[1][0]) - 1
+
+	c0 := counters[0][0]
+	c1 := counters[1][0]
+
+	for i := 1; i < 256; i++ {
+		c0 += uint16(byte_counters[0][i])
+		c1 += uint16(byte_counters[1][i])
+
+		counters[0][i] = c0
+		counters[1][i] = c1
+	}
+
+	counters0 := counters[0][:]
+	for i := int(N); i >= 1; i-- {
+		byte0 := uint32(v[i-1])
+		byte1 := uint32(v[i]) // here we can access extra byte from input array so make sure its zero
+		tmp_indices[counters0[v[i]]] = byte0<<24 | byte1<<16 | uint32(i-1)
+		counters0[v[i]]--
+	}
+
+	counters1 := counters[1][:]
+	_ = tmp_indices[N-1]
+	for i := int(N - 1); i >= 0; i-- {
+		data := tmp_indices[i]
+		tmp := counters1[data>>24]
+		counters1[data>>24]--
+		indices[tmp] = data
+	}
+
+	for i := 1; i < int(N); i++ { // no BC here
+		if indices[i-1]&0xffff0000 == indices[i]&0xffff0000 {
+			fix(v, indices, i-1)
+		}
+	}
+
+	// after fixing, convert indices to output
+	_ = output[N]
+	for i, c := range indices[:N] {
+		output[i] = uint16(c)
+	}
+}

astrobwt/astrobwt_fast/astrobwt_optimized_test.go

@@ -0,0 +1,50 @@
+package astrobwt_fast
+
+import "crypto/rand"
+
+//import "strings"
+import "testing"
+
+//import "encoding/hex"
+
+import "github.com/deroproject/derohe/astrobwt"
+
+func TestPOW_optimized_v1(t *testing.T) {
+	scratch := Pool.Get().(*ScratchData)
+
+	for i := 0; i < 4000; i++ {
+		buf := make([]byte, 400, 400)
+		rand.Read(buf)
+
+		expected_output := astrobwt.POW16(buf[:])
+		actual_output := POW_optimized(buf[:], scratch)
+
+		if string(expected_output[:]) != string(actual_output[:]) {
+			t.Fatalf("Test failed: POW and POW_optimized returns different for i=%d buf %x", i, buf)
+		}
+
+	}
+}
+
+func BenchmarkFastSA(t *testing.B) {
+	var buf [stage1_length + 40]byte
+	rand.Read(buf[:])
+	var output [10000]uint16
+
+	scratch := Pool.Get().(*ScratchData)
+
+	rand.Read(buf[:stage1_length])
+	for i := 0; i < t.N; i++ {
+		sort_indices(stage1_length, buf[:], output[:], scratch)
+		// t.Logf("Ran algo %d", i)
+	}
+}
+
+func BenchmarkPOW_optimized(t *testing.B) {
+	var buf [128]byte
+	rand.Read(buf[:])
+	scratch := Pool.Get().(*ScratchData)
+	for i := 0; i < t.N; i++ {
+		_ = POW_optimized(buf[:], scratch)
+	}
+}

astrobwt/astrobwt_fast/endian_big.go

@@ -0,0 +1,9 @@
+//go:build armbe || arm64be || mips || mips64 || ppc64 || s390 || s390x || sparc || sparc64
+// +build armbe arm64be mips mips64 ppc64 s390 s390x sparc sparc64
+
+package astrobwt_fast
+
+const LittleEndian = false
+const BigEndian = true
+
+// see https://github.com/golang/go/blob/master/src/go/build/syslist.go

astrobwt/astrobwt_fast/endian_little.go

@@ -0,0 +1,9 @@
+//go:build amd64 || amd64p32 || 386 || arm || arm64 || mipsle || mips64le || mips64p32le || ppc64le || riscv || riscv64 || wasm || loong64
+// +build amd64 amd64p32 386 arm arm64 mipsle mips64le mips64p32le ppc64le riscv riscv64 wasm loong64
+
+package astrobwt_fast
+
+const LittleEndian = true
+const BigEndian = false
+
+//see https://github.com/golang/go/blob/master/src/go/build/syslist.go

blockchain/blockchain.go

@@ -209,7 +209,7 @@ func Blockchain_Start(params map[string]interface{}) (*Blockchain, error) {
 		logger.Info("Chain Pruned till", "topoheight", chain.Pruned)
 	}
 
-	// detect case if chain was corrupted earlier,so as it can be deleted and resynced 
+	// detect case if chain was corrupted earlier,so as it can be deleted and resynced
 	if chain.Pruned < globals.Config.HF1_HEIGHT && globals.IsMainnet() && chain.Get_Height() >= globals.Config.HF1_HEIGHT+1 {
 		toporecord, err := chain.Store.Topo_store.Read(globals.Config.HF1_HEIGHT + 1)
 		if err != nil {

build_all.sh

@@ -16,7 +16,7 @@ bash $ABSDIR/build_package.sh "./cmd/derod"
 bash $ABSDIR/build_package.sh "./cmd/explorer"
 bash $ABSDIR/build_package.sh "./cmd/dero-wallet-cli"
 bash $ABSDIR/build_package.sh "./cmd/dero-miner"
-#bash $ABSDIR/build_package.sh "./cmd/simulator"
+bash $ABSDIR/build_package.sh "./cmd/simulator"
 #bash $ABSDIR/build_package.sh "./cmd/rpc_examples/pong_server"
 
 
@@ -36,8 +36,8 @@ go run github.com/randall77/makefat ./dero_darwin_universal/derod-darwin  ./dero
 go run github.com/randall77/makefat ./dero_darwin_universal/explorer-darwin  ./dero_darwin_amd64/explorer-darwin-amd64 ./dero_darwin_arm64/explorer-darwin-arm64
 go run github.com/randall77/makefat ./dero_darwin_universal/dero-wallet-cli-darwin  ./dero_darwin_amd64/dero-wallet-cli-darwin-amd64 ./dero_darwin_arm64/dero-wallet-cli-darwin-arm64
 go run github.com/randall77/makefat ./dero_darwin_universal/dero-miner-darwin  ./dero_darwin_amd64/dero-miner-darwin-amd64 ./dero_darwin_arm64/dero-miner-darwin-arm64
-#go run github.com/randall77/makefat ./dero_darwin_universal/simulator-darwin  ./dero_darwin_amd64/simulator-darwin-amd64 ./dero_darwin_arm64/simulator-darwin-arm64
+go run github.com/randall77/makefat ./dero_darwin_universal/simulator-darwin  ./dero_darwin_amd64/simulator-darwin-amd64 ./dero_darwin_arm64/simulator-darwin-arm64
-go run github.com/randall77/makefat ./dero_darwin_universal/pong_server-darwin  ./dero_darwin_amd64/pong_server-darwin-amd64 ./dero_darwin_arm64/pong_server-darwin-arm64
+#go run github.com/randall77/makefat ./dero_darwin_universal/pong_server-darwin  ./dero_darwin_amd64/pong_server-darwin-amd64 ./dero_darwin_arm64/pong_server-darwin-arm64
 
 rm -rf dero_darwin_amd64
 rm -rf dero_darwin_arm64

cmd/dero-miner/miner.go

@@ -46,7 +46,7 @@ import "github.com/deroproject/derohe/rpc"
 import "github.com/chzyer/readline"
 import "github.com/docopt/docopt-go"
 
-import "github.com/deroproject/derohe/pow"
+import "github.com/deroproject/derohe/astrobwt/astrobwt_fast"
 
 import "github.com/gorilla/websocket"
 
@@ -75,8 +75,8 @@ ONE CPU, ONE VOTE.
 http://wiki.dero.io
 
 Usage:
-  dero-miner  --wallet-address=<wallet_address> [--daemon-rpc-address=<127.0.0.1:10102>] [--mining-threads=<threads>] [--testnet] [--debug]
+  dero-miner  --wallet-address=<wallet_address> [--daemon-rpc-address=<minernode1.dero.live:10100>] [--mining-threads=<threads>] [--testnet] [--debug]
-  dero-miner --bench [--max-pow-size=1120]
+  dero-miner --bench 
   dero-miner -h | --help
   dero-miner --version
 
@@ -84,12 +84,12 @@ Options:
   -h --help     Show this screen.
   --version     Show version.
   --bench  	    Run benchmark mode.
-  --daemon-rpc-address=<127.0.0.1:10102>    Miner will connect to daemon RPC on this port.
+  --daemon-rpc-address=<127.0.0.1:10102>    Miner will connect to daemon RPC on this port (default minernode1.dero.live:10100).
   --wallet-address=<wallet_address>    This address is rewarded when a block is mined sucessfully.
   --mining-threads=<threads>         Number of CPU threads for mining [default: ` + fmt.Sprintf("%d", runtime.GOMAXPROCS(0)) + `]
 
-Example Mainnet: ./dero-miner-linux-amd64 --wallet-address dero1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqhmy4zf --daemon-rpc-address=http://explorer.dero.io:10102 
+Example Mainnet: ./dero-miner-linux-amd64 --wallet-address dero1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqhmy4zf --daemon-rpc-address=minernode1.dero.live:10100
-Example Testnet: ./dero-miner-linux-amd64 --wallet-address deto1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqdzcd8p --daemon-rpc-address=http://127.0.0.1:40402 
+Example Testnet: ./dero-miner-linux-amd64 --wallet-address deto1qy0ehnqjpr0wxqnknyc66du2fsxyktppkr8m8e6jvplp954klfjz2qqdzcd8p --daemon-rpc-address=127.0.0.1:40402 
 If daemon running on local machine no requirement of '--daemon-rpc-address' argument. 
 `
 var Exit_In_Progress = make(chan bool)
@@ -154,7 +154,7 @@ func main() {
 	}
 
 	if !globals.Arguments["--testnet"].(bool) {
-		daemon_rpc_address = "127.0.0.1:10100"
+		daemon_rpc_address = "minernode1.dero.live:10100"
 	} else {
 		daemon_rpc_address = "127.0.0.1:10100"
 	}
@@ -379,10 +379,11 @@ func random_execution(wg *sync.WaitGroup, iterations int) {
 	runtime.LockOSThread()
 	//threadaffinity()
 
+	scratch := astrobwt_fast.Pool.Get().(*astrobwt_fast.ScratchData)
 	rand.Read(workbuf[:])
 
 	for i := 0; i < iterations; i++ {
-		_ = pow.Pow(workbuf[:])
+		_ = astrobwt_fast.POW_optimized(workbuf[:], scratch)
 	}
 	wg.Done()
 	runtime.UnlockOSThread()
@@ -446,7 +447,9 @@ func mineblock(tid int) {
 	var diff big.Int
 	var work [block.MINIBLOCK_SIZE]byte
 
-    time.Sleep(5*time.Second)
+	scratch := astrobwt_fast.Pool.Get().(*astrobwt_fast.ScratchData)
+
+	time.Sleep(5 * time.Second)
 
 	nonce_buf := work[block.MINIBLOCK_SIZE-5:] //since slices are linked, it modifies parent
 	runtime.LockOSThread()
@@ -483,7 +486,7 @@ func mineblock(tid int) {
 			i++
 			binary.BigEndian.PutUint32(nonce_buf, i)
 
-			powhash := pow.Pow(work[:])
+			powhash := astrobwt_fast.POW_optimized(work[:], scratch)
 			atomic.AddUint64(&counter, 1)
 
 			if CheckPowHashBig(powhash, &diff) == true {

config/version.go

@@ -20,4 +20,4 @@ import "github.com/blang/semver/v4"
 
 // right now it has to be manually changed
 // do we need to include git commitsha??
-var Version = semver.MustParse("3.4.133-48.DEROHE.STARGATE+26022022")
+var Version = semver.MustParse("3.4.138-49.DEROHE.STARGATE+10032022")

walletapi/wallet_disk.go

@@ -138,6 +138,10 @@ func (w *Wallet_Disk) Save_Wallet() (err error) {
 		return
 	}
 
+	// attempt to protect atleast one backup copy
+	os.Remove(w.filename + ".bak")
+	os.Rename(w.filename, w.filename+".bak") // renames are atomic, thus more chances of retention
+
 	return ioutil.WriteFile(w.filename, w.Wallet_Memory.db_memory, 0600)
 }
 

walletapi/wallet_transfer.go

@@ -203,6 +203,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint
 		// try to resolve name to address here
 		if _, err = rpc.NewAddress(transfers[t].Destination); err != nil {
 			if transfers[t].Destination, err = w.NameToAddress(transfers[t].Destination); err != nil {
+				err = fmt.Errorf("could not decode name or address err '%s' name '%s'\n", err, transfers[t].Destination)
 				return
 			}
 		}
@@ -225,6 +226,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint
 	// we currently bypass this since random members are chosen which have not been used in last 5 block
 	_, noncetopo, block_hash, self_e, err := w.GetEncryptedBalanceAtTopoHeight(zeroscid, -1, w.GetAddress().String())
 	if err != nil {
+		err = fmt.Errorf("could not obtain encrypted balance for self err %s\n", err)
 		return
 	}
 
@@ -242,6 +244,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, ringsize uint
 
 	er, err := w.GetSelfEncryptedBalanceAtTopoHeight(transfers[0].SCID, topoheight)
 	if err != nil {
+		err = fmt.Errorf("could not obtain encrypted balance for self err %s\n", err)
 		return
 	}
 	height := uint64(er.Height)