diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index 2783673..02fdeee 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -49,7 +49,6 @@ import hashicorp_lru "github.com/hashicorp/golang-lru" import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/cryptography/crypto" import "github.com/deroproject/derohe/errormsg" -import "github.com/prometheus/client_golang/prometheus" //import "github.com/deroproject/derosuite/address" import "github.com/deroproject/derohe/block" @@ -58,6 +57,7 @@ import "github.com/deroproject/derohe/transaction" import "github.com/deroproject/derohe/blockchain/mempool" import "github.com/deroproject/derohe/blockchain/regpool" import "github.com/deroproject/derohe/rpc" +import "github.com/deroproject/derohe/metrics" /* import "github.com/deroproject/derosuite/emission" @@ -67,7 +67,7 @@ import "github.com/deroproject/derosuite/crypto/ringct" import "github.com/deroproject/derosuite/checkpoints" -import "github.com/deroproject/derosuite/metrics" + import "github.com/deroproject/derosuite/blockchain/inputmaturity" */ @@ -276,16 +276,7 @@ func Blockchain_Start(params map[string]interface{}) (*Blockchain, error) { */ - /* - // register the metrics with the metrics registry - metrics.Registry.MustRegister(blockchain_tx_counter) - metrics.Registry.MustRegister(mempool_tx_counter) - metrics.Registry.MustRegister(mempool_tx_count) - metrics.Registry.MustRegister(block_size) - metrics.Registry.MustRegister(transaction_size) - metrics.Registry.MustRegister(block_tx_count) - metrics.Registry.MustRegister(block_processing_time) - */ + atomic.AddUint32(&globals.Subsystem_Active, 1) // increment subsystem return &chain, nil @@ -334,20 +325,24 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro } }() - blockchain_tx_counter.Add(float64(len(cbl.Bl.Tx_hashes))) - block_tx_count.Observe(float64(len(cbl.Bl.Tx_hashes))) - block_processing_time.Observe(float64(time.Now().Sub(processing_start).Round(time.Millisecond) / 1000000)) - - // tracks counters for tx_size + metrics.Blockchain_tx_counter.Add(len(cbl.Bl.Tx_hashes)) + metrics.Block_tx.Update(float64(len(cbl.Bl.Tx_hashes))) + metrics.Block_processing_time.UpdateDuration(processing_start) + // tracks counters for tx internals, do we need to serialize everytime, just for stats { complete_block_size := 0 for i := 0; i < len(cbl.Txs); i++ { tx_size := len(cbl.Txs[i].Serialize()) complete_block_size += tx_size - transaction_size.Observe(float64(tx_size)) + metrics.Transaction_size.Update(float64(tx_size)) + + metrics.Transaction_outputs.Update(float64(len(cbl.Txs[i].Payloads))) + if len(cbl.Txs[i].Payloads) >= 1 { + metrics.Transaction_ring_size.Update(float64(cbl.Txs[i].Payloads[0].Statement.RingSize)) + } } - block_size.Observe(float64(complete_block_size)) + metrics.Block_size.Update(float64(complete_block_size)) } }() @@ -586,49 +581,73 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro } } - // another check, whether the tx is build with the latest snapshot of balance tree + + // another check, whether the tx is build with the correct height reference { for i := 0; i < len(cbl.Txs); i++ { if cbl.Txs[i].TransactionType == transaction.NORMAL || cbl.Txs[i].TransactionType == transaction.BURN_TX || cbl.Txs[i].TransactionType == transaction.SC_TX { - if cbl.Txs[i].Height+1 != cbl.Bl.Height { + if cbl.Txs[i].Height % config.BLOCK_BATCH_SIZE != 0 { + block_logger.Warnf("invalid tx mined %s tx height not a multiple of %d", cbl.Txs[i].GetHash(), config.BLOCK_BATCH_SIZE) + return errormsg.ErrTXDoubleSpend, false + } + + if cbl.Txs[i].Height >= cbl.Bl.Height { // chain height should be more than tx block_logger.Warnf("invalid tx mined %s", cbl.Txs[i].GetHash()) return errormsg.ErrTXDoubleSpend, false } + if !Verify_Transaction_NonCoinbase_Height(cbl.Txs[i],cbl.Bl.Height ){ + block_logger.Warnf("invalid tx mined height issue %s %d %d", cbl.Txs[i].GetHash(),cbl.Txs[i].Height,cbl.Bl.Height ) + return errormsg.ErrTXDoubleSpend, false + } } } } + // another check, whether the tx contains any duplicate nonces within the block // block wide duplicate input detector { nonce_map := map[crypto.Hash]bool{} for i := 0; i < len(cbl.Txs); i++ { - if cbl.Txs[i].TransactionType == transaction.NORMAL || cbl.Txs[i].TransactionType == transaction.BURN_TX || cbl.Txs[i].TransactionType == transaction.SC_TX { - if _, ok := nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce()]; ok { - block_logger.Warnf("Double Spend attack within block %s", cbl.Txs[i].GetHash()) + if _, ok := nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce1()]; ok { + block_logger.Warnf("Double Spend attack within block nonce1 %s", cbl.Txs[i].GetHash()) return errormsg.ErrTXDoubleSpend, false } - nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce()] = true + nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce1()] = true + + if _, ok := nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce2()]; ok { + block_logger.Warnf("Double Spend attack within block nonce2 %s", cbl.Txs[i].GetHash()) + return errormsg.ErrTXDoubleSpend, false + } + nonce_map[cbl.Txs[i].Payloads[0].Proof.Nonce2()] = true } } } // we also need to reject if the the immediately reachable history, has spent the nonce // both the checks works on the basis of nonces and not on the basis of txhash - /* { - reachable_nonces := chain.BuildReachabilityNonces(bl) + reachable_nonces,err := chain.BuildNonces(bl.Tips) + if err != nil{ + return err,false + } for i := 0; i < len(cbl.Txs); i++ { // loop through all the TXs - if cbl.Txs[i].TransactionType == transaction.NORMAL { - if _, ok := reachable_nonces[cbl.Txs[i].Proof.Nonce()]; ok { + switch cbl.Txs[i].TransactionType { + case transaction.PREMINE,transaction.REGISTRATION, transaction.COINBASE: // these donot have nonces + + case transaction.NORMAL, transaction.BURN_TX, transaction.SC_TX: // these have nonces + + if reachable_nonces[cbl.Txs[i].Payloads[0].Proof.Nonce1()] || reachable_nonces[cbl.Txs[i].Payloads[0].Proof.Nonce1()] { block_logger.Warnf("Double spend attack tx %s is already mined, rejecting ", cbl.Txs[i].GetHash()) return errormsg.ErrTXDoubleSpend, false } + default: } } - }*/ + } + // we need to anyways verify the TXS since proofs are not covered by checksum { @@ -729,11 +748,53 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro base_topo_index = 0 } + nonce_map := map[crypto.Hash]bool{} + + // any blocks which have not changed their topo will be skipped using graviton trick skip := true for i := int64(0); i < int64(len(full_order)); i++ { + if i == 0 { + bl_prev, err1 := chain.Load_BL_FROM_ID(full_order[0]) + if err1 != nil { + block_logger.Debugf("Cannot load block %s for client protocol,probably DB/disk corruption", full_order[0]) + return errormsg.ErrInvalidBlock, false + } + nonce_map,err = chain.BuildNonces(bl_prev.Tips) + if err != nil { + panic(err) + } + } + + if i >= 1 { // load nonces for previous blocks + bl_prev_hash := full_order[i-1] + bl_prev, err1 := chain.Load_BL_FROM_ID(full_order[i-1]) + if err1 != nil { + block_logger.Debugf("Cannot load block %s for client protocol,probably DB/disk corruption", bl_prev_hash) + return errormsg.ErrInvalidBlock, false + } + + for i := 0; i < len(bl_prev.Tx_hashes); i++ { // load all tx one by one, no skipping + tx_bytes, err := chain.Store.Block_tx_store.ReadTX(bl_prev.Tx_hashes[i]) + if err != nil { + panic(fmt.Errorf("Cannot load tx for %s err %s", bl_prev.Tx_hashes[i], err)) + } + + var tx transaction.Transaction + if err = tx.DeserializeHeader(tx_bytes); err != nil { + panic(err) + } + + if !tx.IsRegistration(){ + // tx has been loaded, now lets get the nonce + nonce_map[tx.Payloads[0].Proof.Nonce1()] = true // add element to map for next check + nonce_map[tx.Payloads[0].Proof.Nonce2()] = true // add element to map for next check + } + } + } + // check whether the new block is at the same position at the last position current_topo_block := i + base_topo_index previous_topo_block := current_topo_block - 1 @@ -761,7 +822,7 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro bl_current_hash := full_order[i] bl_current, err1 := chain.Load_BL_FROM_ID(bl_current_hash) if err1 != nil { - block_logger.Debugf("Cannot load block %s for client protocol,probably DB corruption", bl_current_hash) + block_logger.Debugf("Cannot load block %s for client protocol,probably DB/disk corruption", bl_current_hash) return errormsg.ErrInvalidBlock, false } @@ -770,9 +831,6 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro height_current := chain.Calculate_Height_At_Tips(bl_current.Tips) hard_fork_version_current := chain.Get_Current_Version_at_Height(height_current) - // this version does not require client protocol as of now - // run full client protocol and find valid transactions - // rlog.Debugf("running client protocol for %s minertx %s topo %d", bl_current_hash, bl_current.Miner_TX.GetHash(), highest_topo) // generate miner TX rewards as per client protocol if hard_fork_version_current == 1 { @@ -780,7 +838,6 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro } var balance_tree, sc_meta *graviton.Tree - _ = sc_meta var ss *graviton.Snapshot if bl_current.Height == 0 { // if it's genesis block @@ -818,14 +875,14 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro fees_collected := uint64(0) - // side blocks only represent chain strenth , else they are are ignored + // side blocks not only represent chain strenth , they also represent extra data // this means they donot get any reward , 0 reward // their transactions are ignored //chain.Store.Topo_store.Write(i+base_topo_index, full_order[i],0, int64(bl_current.Height)) // write entry so as sideblock could work var data_trees []*graviton.Tree - if !chain.isblock_SideBlock_internal(full_order[i], current_topo_block, int64(bl_current.Height)) { + { sc_change_cache := map[crypto.Hash]*graviton.Tree{} // cache entire changes for entire block @@ -837,12 +894,23 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro if err = tx.DeserializeHeader(tx_bytes); err != nil { panic(err) } + + + if tx.TransactionType != transaction.REGISTRATION { + + if nonce_map[tx.Payloads[0].Proof.Nonce1()] || nonce_map[tx.Payloads[0].Proof.Nonce1()] { + continue // skip this tx, since it has been processed earlier + }else{ + nonce_map[tx.Payloads[0].Proof.Nonce1()] = true + nonce_map[tx.Payloads[0].Proof.Nonce2()] = true + } for t := range tx.Payloads { if !tx.Payloads[t].SCID.IsZero() { tree, _ := ss.GetTree(string(tx.Payloads[t].SCID[:])) sc_change_cache[tx.Payloads[t].SCID] = tree } } + } // we have loaded a tx successfully, now lets execute it tx_fees := chain.process_transaction(sc_change_cache, tx, balance_tree) @@ -886,11 +954,17 @@ func (chain *Blockchain) Add_Complete_Block(cbl *block.Complete_Block) (err erro } - chain.process_miner_transaction(bl_current.Miner_TX, bl_current.Height == 0, balance_tree, fees_collected, bl_current.Height) - } else { - rlog.Debugf("this block is a side block block height %d blid %s ", chain.Load_Block_Height(full_order[i]), full_order[i]) - } + // if block is side block, pass all reward to devs, else pass it to original miner + side_block := chain.isblock_SideBlock_internal(full_order[i], current_topo_block, int64(bl_current.Height)) + if side_block{ + rlog.Debugf("this block is a side block block height %d blid %s ", chain.Load_Block_Height(full_order[i]), full_order[i]) + } + chain.process_miner_transaction(bl_current.Miner_TX, bl_current.Height == 0, balance_tree, fees_collected, bl_current.Height,side_block) + + + + } // we are here, means everything is okay, lets commit the update balance tree @@ -1135,53 +1209,6 @@ func (chain *Blockchain) BlockCheckSum(cbl *block.Complete_Block) []byte { return h.Sum(nil) } -// various counters/gauges which track a numer of metrics -// such as number of txs, number of inputs, number of outputs -// mempool total addition, current mempool size -// block processing time etcs - -// Try it once more, this time with a help string. -var blockchain_tx_counter = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "blockchain_tx_counter", - Help: "Number of tx mined", -}) - -var mempool_tx_counter = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "mempool_tx_counter", - Help: "Total number of tx added in mempool", -}) -var mempool_tx_count = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "mempool_tx_count", - Help: "Number of tx in mempool at this point", -}) - -// track block size about 2 MB -var block_size = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "block_size_byte", - Help: "Block size in byte (complete)", - Buckets: prometheus.LinearBuckets(0, 102400, 10), // start block size 0, each 1 KB step, 2048 such buckets . -}) - -// track transaction size upto 500 KB -var transaction_size = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "tx_size_byte", - Help: "TX size in byte", - Buckets: prometheus.LinearBuckets(0, 10240, 16), // start 0 byte, each 1024 byte, 512 such buckets. -}) - -// number of tx per block -var block_tx_count = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "block_tx_count", - Help: "Number of TX in the block", - Buckets: prometheus.LinearBuckets(0, 20, 25), // start 0 byte, each 1024 byte, 1024 such buckets. -}) - -// -var block_processing_time = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "block_processing_time_ms", - Help: "Block processing time milliseconds", - Buckets: prometheus.LinearBuckets(0, 100, 20), // start 0 ms, each 100 ms, 200 such buckets. -}) // this is the only entrypoint for new txs in the chain // add a transaction to MEMPOOL, @@ -1222,7 +1249,7 @@ func (chain *Blockchain) Add_TX_To_Pool(tx *transaction.Transaction) error { } // track counter for the amount of mempool tx - defer mempool_tx_count.Set(float64(len(chain.Mempool.Mempool_List_TX()))) + defer metrics.Mempool_tx_count.Set(uint64(len(chain.Mempool.Mempool_List_TX()))) txhash := tx.GetHash() @@ -1233,9 +1260,9 @@ func (chain *Blockchain) Add_TX_To_Pool(tx *transaction.Transaction) error { } chain_height := uint64(chain.Get_Height()) - if chain_height > tx.Height { - rlog.Tracef(2, "TX %s rejected since chain has already progressed", txhash) - return fmt.Errorf("TX %s rejected since chain has already progressed", txhash) + if !Verify_Transaction_NonCoinbase_Height(tx,chain_height) { + rlog.Tracef(2, "TX %s rejected since tx is too recent or too old", txhash) + return fmt.Errorf("TX %s rejected since tx is too recent or too old", txhash) } // quick check without calculating everything whether tx is in pool, if yes we do nothing @@ -1285,11 +1312,11 @@ func (chain *Blockchain) Add_TX_To_Pool(tx *transaction.Transaction) error { if chain.Mempool.Mempool_Add_TX(tx, 0) { // new tx come with 0 marker rlog.Tracef(2, "Successfully added tx %s to pool", txhash) - mempool_tx_counter.Inc() + metrics.Mempool_tx_counter.Inc() return nil } else { - rlog.Tracef(2, "TX %s rejected by pool", txhash) - return fmt.Errorf("TX %s rejected by pool", txhash) + rlog.Tracef(2, "TX %s rejected by pool by mempool", txhash) + return fmt.Errorf("TX %s rejected by pool by mempool", txhash) } } @@ -1394,14 +1421,18 @@ func (chain *Blockchain) isblock_SideBlock_internal(blid crypto.Hash, block_topo return false } -// this will return the tx combination as valid/invalid + +// this will return the block ids in which tx has been mined // this is not used as core consensus but reports only to user that his tx though in the blockchain is invalid // a tx is valid, if it exist in a block which is not a side block -func (chain *Blockchain) IS_TX_Valid(txhash crypto.Hash) (valid_blid crypto.Hash, invalid_blid []crypto.Hash, valid bool) { +// this is not part of consensus but only for support of explorer +func (chain *Blockchain) IS_TX_Mined(txhash crypto.Hash) (mined_blocks []crypto.Hash, state_block crypto.Hash, state_block_topo int64) { var tx_bytes []byte var err error + state_found := false + if tx_bytes, err = chain.Store.Block_tx_store.ReadTX(txhash); err != nil { return } @@ -1411,11 +1442,24 @@ func (chain *Blockchain) IS_TX_Valid(txhash crypto.Hash) (valid_blid crypto.Hash return } - blids, _ := chain.Store.Topo_store.binarySearchHeight(int64(tx.Height + 1)) + for i := int64(0); i < 2 *config.BLOCK_BATCH_SIZE;i++ { + blids, topos := chain.Store.Topo_store.binarySearchHeight(int64(tx.Height) + i ) - var exist_list []crypto.Hash + for j, blid := range blids { + if !state_found{ // keep finding state + merkle_hash, err := chain.Load_Merkle_Hash(topos[j]) + if err != nil { + panic(err) + } + + if len(tx.Payloads) >= 1 && merkle_hash == tx.Payloads[0].Statement.Roothash { + state_block = blid + state_block_topo = topos[j] + state_found = true + } + + } - for _, blid := range blids { bl, err := chain.Load_BL_FROM_ID(blid) if err != nil { return @@ -1423,123 +1467,19 @@ func (chain *Blockchain) IS_TX_Valid(txhash crypto.Hash) (valid_blid crypto.Hash for _, bltxhash := range bl.Tx_hashes { if bltxhash == txhash { - exist_list = append(exist_list, blid) + mined_blocks = append(mined_blocks, blid) break } } } - for _, blid := range exist_list { - if chain.Isblock_SideBlock(blid) { - invalid_blid = append(invalid_blid, blid) - } else { - valid_blid = blid - valid = true - } - } + + return } -/* - - -// runs the client protocol which includes the following operations -// if any TX are being duplicate or double-spend ignore them -// mark all the valid transactions as valid -// mark all invalid transactions as invalid -// calculate total fees based on valid TX -// we need NOT check ranges/ring signatures here, as they have been done already by earlier steps -func (chain *Blockchain) client_protocol(dbtx storage.DBTX, bl *block.Block, blid crypto.Hash, height int64, topoheight int64) (total_fees uint64) { - // run client protocol for all TXs - for i := range bl.Tx_hashes { - tx, err := chain.Load_TX_FROM_ID(dbtx, bl.Tx_hashes[i]) - if err != nil { - panic(fmt.Errorf("Cannot load tx for %x err %s ", bl.Tx_hashes[i], err)) - } - // mark TX found in this block also for explorer - chain.store_TX_in_Block(dbtx, blid, bl.Tx_hashes[i]) - - // check all key images as double spend, if double-spend detected mark invalid, else consider valid - if chain.Verify_Transaction_NonCoinbase_DoubleSpend_Check(dbtx, tx) { - - chain.consume_keyimages(dbtx, tx, height) // mark key images as consumed - total_fees += tx.RctSignature.Get_TX_Fee() - - chain.Store_TX_Height(dbtx, bl.Tx_hashes[i], topoheight) // link the tx with the topo height - - //mark tx found in this block is valid - chain.mark_TX(dbtx, blid, bl.Tx_hashes[i], true) - - } else { // TX is double spend or reincluded by 2 blocks simultaneously - rlog.Tracef(1,"Double spend TX is being ignored %s %s", blid, bl.Tx_hashes[i]) - chain.mark_TX(dbtx, blid, bl.Tx_hashes[i], false) - } - } - - return total_fees -} - -// this undoes everything that is done by client protocol -// NOTE: this will have any effect, only if client protocol has been run on this block earlier -func (chain *Blockchain) client_protocol_reverse(dbtx storage.DBTX, bl *block.Block, blid crypto.Hash) { - // run client protocol for all TXs - for i := range bl.Tx_hashes { - tx, err := chain.Load_TX_FROM_ID(dbtx, bl.Tx_hashes[i]) - if err != nil { - panic(fmt.Errorf("Cannot load tx for %x err %s ", bl.Tx_hashes[i], err)) - } - // only the valid TX must be revoked - if chain.IS_TX_Valid(dbtx, blid, bl.Tx_hashes[i]) { - chain.revoke_keyimages(dbtx, tx) // mark key images as not used - - chain.Store_TX_Height(dbtx, bl.Tx_hashes[i], -1) // unlink the tx with the topo height - - //mark tx found in this block is invalid - chain.mark_TX(dbtx, blid, bl.Tx_hashes[i], false) - - } else { // TX is double spend or reincluded by 2 blocks simultaneously - // invalid tx is related - } - } - - return -} - -// scavanger for transactions from rusty/stale tips to reinsert them into pool -func (chain *Blockchain) transaction_scavenger(dbtx storage.DBTX, blid crypto.Hash) { - defer func() { - if r := recover(); r != nil { - logger.Warnf("Recovered while transaction scavenging, Stack trace below ") - logger.Warnf("Stack trace \n%s", debug.Stack()) - } - }() - - logger.Debugf("scavenging transactions from blid %s", blid) - reachable_blocks := chain.BuildReachableBlocks(dbtx, []crypto.Hash{blid}) - reachable_blocks[blid] = true // add self - for k, _ := range reachable_blocks { - if chain.Is_Block_Orphan(k) { - bl, err := chain.Load_BL_FROM_ID(dbtx, k) - if err == nil { - for i := range bl.Tx_hashes { - tx, err := chain.Load_TX_FROM_ID(dbtx, bl.Tx_hashes[i]) - if err != nil { - rlog.Warnf("err while scavenging blid %s txid %s err %s", k, bl.Tx_hashes[i], err) - } else { - // add tx to pool, it will do whatever is necessarry - chain.Add_TX_To_Pool(tx) - } - } - } else { - rlog.Warnf("err while scavenging blid %s err %s", k, err) - } - } - } -} - -*/ // Finds whether a block is orphan // since we donot store any fields, we need to calculate/find the block as orphan // using an algorithm @@ -1548,14 +1488,6 @@ func (chain *Blockchain) Is_Block_Orphan(hash crypto.Hash) bool { return !chain.Is_Block_Topological_order(hash) } -// this is used to find if a tx is orphan, YES orphan TX -// these can occur during when they lie only in a side block -// so the TX becomes orphan ( chances are less may be less that .000001 % but they are there) -// if a tx is not valid in any of the blocks, it has been mined it is orphan -func (chain *Blockchain) Is_TX_Orphan(hash crypto.Hash) (result bool) { - _, _, result = chain.IS_TX_Valid(hash) - return !result -} // verifies whether we are lagging // return true if we need resync @@ -1628,6 +1560,74 @@ func (chain *Blockchain) Rewind_Chain(rewind_count int) (result bool) { return true } + +// used to check for nonce duplication, so as txs can avoid reexecution +// note that the tips might not be part of chain +func (chain *Blockchain) BuildNonces(tips []crypto.Hash) (map[crypto.Hash]bool, error) { + nonce_list := map[crypto.Hash]bool{} // contains a list of all reachable blocks + + var blocks_map = map[crypto.Hash]bool{} + var blocks_pending []crypto.Hash + for i := range tips { + blocks_pending = append(blocks_pending,tips[i]) + } + + height := chain.Calculate_Height_At_Tips(tips) // we are 1 higher than previous highest tip + + + for ; len(blocks_pending) > 0; { + + blid := blocks_pending[len(blocks_pending)-1] + blocks_pending = blocks_pending[:len(blocks_pending)-1] + + if _, processed := blocks_map[blid]; processed { + continue + } + blocks_map[blid] = true + + bl, err := chain.Load_BL_FROM_ID(blid) + if err != nil { + return nil,err + } + + if bl.Height == 0 { // if we reach genesis, we skip + continue + } + + if int64(bl.Height + (config.BLOCK_BATCH_SIZE*4)) < height { // if we are too deep, skip past + continue + } + + blocks_pending = append(blocks_pending,bl.Tips...) + + + for i := 0; i < len(bl.Tx_hashes); i++ { // load all tx one by one, no skipping + tx_bytes, err := chain.Store.Block_tx_store.ReadTX(bl.Tx_hashes[i]) + if err != nil { + panic(fmt.Errorf("Cannot load tx for %s err %s", bl.Tx_hashes[i], err)) + } + + var tx transaction.Transaction + if err = tx.DeserializeHeader(tx_bytes); err != nil { + panic(err) + } + + if !tx.IsRegistration(){ + + // tx has been loaded, now lets get the nonce + nonce_list[tx.Payloads[0].Proof.Nonce1()] = true // add element to map for next check + nonce_list[tx.Payloads[0].Proof.Nonce2()] = true // add element to map for next check + } + + } + + + } + + return nonce_list,nil +} + + // build reachability graph upto 2*config deeps to answer reachability queries func (chain *Blockchain) buildReachability_internal(reachmap map[crypto.Hash]bool, blid crypto.Hash, level int) { bl, err := chain.Load_BL_FROM_ID(blid) @@ -1731,7 +1731,8 @@ func (chain *Blockchain) BuildReachabilityNonces(bl *block.Block) map[crypto.Has } // tx has been loaded, now lets get the nonce - nonce_reach_map[tx.Payloads[0].Proof.Nonce()] = true // add element to map for next check + nonce_reach_map[tx.Payloads[0].Proof.Nonce1()] = true // add element to map for next check + nonce_reach_map[tx.Payloads[0].Proof.Nonce2()] = true // add element to map for next check } } return nonce_reach_map diff --git a/blockchain/mempool/mempool.go b/blockchain/mempool/mempool.go index f5c3d63..a49e720 100644 --- a/blockchain/mempool/mempool.go +++ b/blockchain/mempool/mempool.go @@ -31,6 +31,7 @@ import log "github.com/sirupsen/logrus" import "github.com/deroproject/derohe/transaction" import "github.com/deroproject/derohe/globals" +import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/cryptography/crypto" // this is only used for sorting and nothing else @@ -202,7 +203,7 @@ func (pool *Mempool) HouseKeeping(height uint64) { pool.txs.Range(func(k, value interface{}) bool { txhash := k.(crypto.Hash) v := value.(*mempool_object) - if height >= (v.Tx.Height + 1) { // if we have moved 1 heights, chances are reorg are almost nil + if height >= (v.Tx.Height + 3* config.BLOCK_BATCH_SIZE + 1) { // if we have moved 1 heights, chances are reorg are almost nil delete_list = append(delete_list, txhash) } return true @@ -283,7 +284,12 @@ func (pool *Mempool) Mempool_Add_TX(tx *transaction.Transaction, Height uint64) var object mempool_object tx_hash := crypto.Hash(tx.GetHash()) - if pool.Mempool_Keyimage_Spent(tx.Payloads[0].Proof.Nonce()) { + if pool.Mempool_Keyimage_Spent(tx.Payloads[0].Proof.Nonce1()) { + rlog.Debugf("Rejecting TX, since nonce already seen %x", tx_hash) + return false + } + + if pool.Mempool_Keyimage_Spent(tx.Payloads[0].Proof.Nonce2()) { rlog.Debugf("Rejecting TX, since nonce already seen %x", tx_hash) return false } @@ -300,7 +306,8 @@ func (pool *Mempool) Mempool_Add_TX(tx *transaction.Transaction, Height uint64) // pool.key_images.Store(tx.Vin[i].(transaction.Txin_to_key).K_image,true) // add element to map for next check // } - pool.key_images.Store(tx.Payloads[0].Proof.Nonce(), true) + pool.key_images.Store(tx.Payloads[0].Proof.Nonce1(), true) + pool.key_images.Store(tx.Payloads[0].Proof.Nonce2(), true) // we are here means we can add it to pool object.Tx = tx @@ -367,7 +374,8 @@ func (pool *Mempool) Mempool_Delete_TX(txid crypto.Hash) (tx *transaction.Transa // for i := 0; i < len(object.Tx.Vin); i++ { // pool.key_images.Delete(object.Tx.Vin[i].(transaction.Txin_to_key).K_image) // } - pool.key_images.Delete(tx.Payloads[0].Proof.Nonce()) + pool.key_images.Delete(tx.Payloads[0].Proof.Nonce1()) + pool.key_images.Delete(tx.Payloads[0].Proof.Nonce2()) //pool.sort_list() // sort and update pool list pool.modified = true // pool has been modified diff --git a/blockchain/miner_block.go b/blockchain/miner_block.go index d362a24..b1c68de 100644 --- a/blockchain/miner_block.go +++ b/blockchain/miner_block.go @@ -74,7 +74,6 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t if err != nil { panic(err) } - } topoheight := chain.Load_TOPO_HEIGHT() @@ -116,6 +115,15 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t sizeoftxs := uint64(0) // size of all non coinbase tx included within this block //fees_collected := uint64(0) + + nonce_map,err := chain.BuildNonces(bl.Tips) + if err != nil { + panic(err) + } + + local_nonce_map := map[crypto.Hash]bool{} + + _ = sizeoftxs // add upto 100 registration tx each registration tx is 99 bytes, so 100 tx will take 9900 bytes or 10KB @@ -129,9 +137,8 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t _, err = balance_tree.Get(tx.MinerAddress[:]) if err != nil { if xerrors.Is(err, graviton.ErrNotFound) { // address needs registration - cbl.Txs = append(cbl.Txs, tx) - tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i]) + tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i]) } else { panic(err) } @@ -159,7 +166,7 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t } tx := chain.Mempool.Mempool_Get_TX(tx_hash_list_sorted[i].Hash) - if tx != nil && int64(tx.Height)+1 == height { + if tx != nil && Verify_Transaction_NonCoinbase_Height(tx,uint64(height)) { /* // skip and delete any mempool tx @@ -180,10 +187,22 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t continue } */ - rlog.Tracef(1, "Adding Top Sorted tx %s to Complete_Block current size %.2f KB max possible %.2f KB\n", tx_hash_list_sorted[i].Hash, float32(sizeoftxs+tx_hash_list_sorted[i].Size)/1024.0, float32(config.STARGATE_HE_MAX_BLOCK_SIZE)/1024.0) - sizeoftxs += tx_hash_list_sorted[i].Size - cbl.Txs = append(cbl.Txs, tx) - tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i].Hash) + + + if nonce_map[tx.Payloads[0].Proof.Nonce1()] || nonce_map[tx.Payloads[0].Proof.Nonce1()] || + local_nonce_map[tx.Payloads[0].Proof.Nonce1()] || local_nonce_map[tx.Payloads[0].Proof.Nonce1()] { + continue // skip this tx + } + + cbl.Txs = append(cbl.Txs, tx) + tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i].Hash) + local_nonce_map[tx.Payloads[0].Proof.Nonce1()] = true + local_nonce_map[tx.Payloads[0].Proof.Nonce2()] = true + rlog.Tracef(1, "Adding Top Sorted tx %s to Complete_Block current size %.2f KB max possible %.2f KB\n", tx_hash_list_sorted[i].Hash, float32(sizeoftxs+tx_hash_list_sorted[i].Size)/1024.0, float32(config.STARGATE_HE_MAX_BLOCK_SIZE)/1024.0) + sizeoftxs += tx_hash_list_sorted[i].Size + + + } } // any left over transactions, should be randomly selected @@ -204,12 +223,21 @@ func (chain *Blockchain) Create_new_miner_block(miner_address rpc.Address, tx *t } tx := chain.Mempool.Mempool_Get_TX(tx_hash_list_sorted[i].Hash) - if tx != nil && int64(tx.Height)+1 == height { + if tx != nil && Verify_Transaction_NonCoinbase_Height(tx, uint64(height)){ - rlog.Tracef(1, "Adding Random tx %s to Complete_Block current size %.2f KB max possible %.2f KB\n", tx_hash_list_sorted[i].Hash, float32(sizeoftxs+tx_hash_list_sorted[i].Size)/1024.0, float32(config.STARGATE_HE_MAX_BLOCK_SIZE)/1024.0) - sizeoftxs += tx_hash_list_sorted[i].Size - cbl.Txs = append(cbl.Txs, tx) - tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i].Hash) + + if nonce_map[tx.Payloads[0].Proof.Nonce1()] || nonce_map[tx.Payloads[0].Proof.Nonce1()] || + local_nonce_map[tx.Payloads[0].Proof.Nonce1()] || local_nonce_map[tx.Payloads[0].Proof.Nonce1()] { + continue // skip this tx + } + + cbl.Txs = append(cbl.Txs, tx) + tx_hash_list_included = append(tx_hash_list_included, tx_hash_list_sorted[i].Hash) + local_nonce_map[tx.Payloads[0].Proof.Nonce1()] = true + local_nonce_map[tx.Payloads[0].Proof.Nonce2()] = true + rlog.Tracef(1, "Adding Random tx %s to Complete_Block current size %.2f KB max possible %.2f KB\n", tx_hash_list_sorted[i].Hash, float32(sizeoftxs+tx_hash_list_sorted[i].Size)/1024.0, float32(config.STARGATE_HE_MAX_BLOCK_SIZE)/1024.0) + + sizeoftxs += tx_hash_list_sorted[i].Size } } diff --git a/blockchain/transaction_execute.go b/blockchain/transaction_execute.go index 6f47e61..424e331 100644 --- a/blockchain/transaction_execute.go +++ b/blockchain/transaction_execute.go @@ -54,7 +54,7 @@ func CalcBlockReward(height uint64) uint64 { } // process the miner tx, giving fees, miner rewatd etc -func (chain *Blockchain) process_miner_transaction(tx transaction.Transaction, genesis bool, balance_tree *graviton.Tree, fees uint64, height uint64) { +func (chain *Blockchain) process_miner_transaction(tx transaction.Transaction, genesis bool, balance_tree *graviton.Tree, fees uint64, height uint64,sideblock bool) { var acckey crypto.Point if err := acckey.DecodeCompressed(tx.MinerAddress[:]); err != nil { panic(err) @@ -69,29 +69,26 @@ func (chain *Blockchain) process_miner_transaction(tx transaction.Transaction, g // general coin base transaction base_reward := CalcBlockReward(uint64(height)) - full_reward := base_reward + fees + full_reward := base_reward+ fees - dev_reward := (full_reward * config.DEVSHARE) / 10000 // take % from reward - miner_reward := full_reward - dev_reward // it's value, do subtraction - { // giver miner reward - balance_serialized, err := balance_tree.Get(tx.MinerAddress[:]) - if err != nil { - panic(err) - } - balance := new(crypto.ElGamal).Deserialize(balance_serialized) - balance = balance.Plus(new(big.Int).SetUint64(miner_reward)) // add miners reward to miners balance homomorphically - balance_tree.Put(tx.MinerAddress[:], balance.Serialize()) // reserialize and store - } - - { // give devs reward + if sideblock {// give devs reward balance_serialized, err := balance_tree.Get(chain.Dev_Address_Bytes[:]) if err != nil { panic(err) } balance := new(crypto.ElGamal).Deserialize(balance_serialized) - balance = balance.Plus(new(big.Int).SetUint64(dev_reward)) // add devs reward to devs balance homomorphically + balance = balance.Plus(new(big.Int).SetUint64(full_reward)) // add devs reward to devs balance homomorphically balance_tree.Put(chain.Dev_Address_Bytes[:], balance.Serialize()) // reserialize and store + + }else{ // giver miner reward + balance_serialized, err := balance_tree.Get(tx.MinerAddress[:]) + if err != nil { + panic(err) + } + balance := new(crypto.ElGamal).Deserialize(balance_serialized) + balance = balance.Plus(new(big.Int).SetUint64(full_reward)) // add miners reward to miners balance homomorphically + balance_tree.Put(tx.MinerAddress[:], balance.Serialize()) // reserialize and store } return @@ -106,6 +103,9 @@ func (chain *Blockchain) process_transaction(changed map[crypto.Hash]*graviton.T switch tx.TransactionType { case transaction.REGISTRATION: + if _, err := balance_tree.Get(tx.MinerAddress[:]); err == nil { + return 0 + } if _, err := balance_tree.Get(tx.MinerAddress[:]); err != nil { if !xerrors.Is(err, graviton.ErrNotFound) { // any other err except not found panic panic(err) diff --git a/blockchain/transaction_verify.go b/blockchain/transaction_verify.go index a62ff01..31fd4bf 100644 --- a/blockchain/transaction_verify.go +++ b/blockchain/transaction_verify.go @@ -113,6 +113,34 @@ func (chain *Blockchain) Verify_Transaction_Coinbase(cbl *block.Complete_Block, return nil // success comes last } + +// only verifies height whether all height checks are good +func Verify_Transaction_NonCoinbase_Height(tx *transaction.Transaction, chain_height uint64) bool { + return Verify_Transaction_Height(tx.Height, chain_height) +} + + +func Verify_Transaction_Height(tx_height, chain_height uint64) bool{ + if tx_height % config.BLOCK_BATCH_SIZE != 0 { + return false + } + + if tx_height >= chain_height { + return false + } + + if chain_height-tx_height <= 5 { // we should be atleast 5 steps from top + return false + } + + comp := (chain_height / config.BLOCK_BATCH_SIZE) - (tx_height / config.BLOCK_BATCH_SIZE) + if comp ==0 || comp ==1 { + return true + }else{ + return false + } +} + // all non miner tx must be non-coinbase tx // each check is placed in a separate block of code, to avoid ambigous code or faulty checks // all check are placed and not within individual functions ( so as we cannot skip a check ) @@ -329,7 +357,7 @@ func (chain *Blockchain) Verify_Transaction_NonCoinbase(hf_version int64, tx *tr // at this point has been completely expanded, verify the tx statement for t := range tx.Payloads { - if !tx.Payloads[t].Proof.Verify(&tx.Payloads[t].Statement, tx.GetHash(), tx.Payloads[t].BurnValue) { + if !tx.Payloads[t].Proof.Verify(&tx.Payloads[t].Statement, tx.GetHash(), tx.Height, tx.Payloads[t].BurnValue) { fmt.Printf("Statement %+v\n", tx.Payloads[t].Statement) fmt.Printf("Proof %+v\n", tx.Payloads[t].Proof) diff --git a/cmd/derod/rpc_dero_getencryptedbalance.go b/cmd/derod/rpc_dero_getencryptedbalance.go index 88dcac7..458e195 100644 --- a/cmd/derod/rpc_dero_getencryptedbalance.go +++ b/cmd/derod/rpc_dero_getencryptedbalance.go @@ -27,6 +27,7 @@ import "github.com/deroproject/derohe/globals" import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/errormsg" import "github.com/deroproject/derohe/rpc" +import "github.com/deroproject/derohe/blockchain" //import "github.com/deroproject/derohe/dvm" //import "github.com/deroproject/derohe/cryptography/crypto" @@ -52,6 +53,37 @@ func (DERO_RPC_APIS) GetEncryptedBalance(ctx context.Context, p rpc.GetEncrypted topoheight = p.TopoHeight } + switch p.TopoHeight { + case rpc.RECENT_BATCH_BLOCK: // give data of specific point from where tx could be built + + chain_height := chain.Get_Height() + + var topo_list []int64 + for ;topoheight > 0; { + toporecord, err := chain.Store.Topo_store.Read(topoheight) + if err != nil { + panic(err) + } + + if blockchain.Verify_Transaction_Height(uint64(toporecord.Height), uint64(chain_height)){ + if chain_height - toporecord.Height <= (3*config.BLOCK_BATCH_SIZE)/2 { // give us enough leeway + topo_list=append(topo_list, topoheight) + } + } + + if chain_height-toporecord.Height >= 2 * config.BLOCK_BATCH_SIZE { + break; + } + + topoheight-- + } + topoheight = topo_list[len(topo_list)-1] + + case rpc.RECENT_BLOCK : fallthrough + default: + + } + toporecord, err := chain.Store.Topo_store.Read(topoheight) if err != nil { panic(err) diff --git a/cmd/derod/rpc_dero_gettransactions.go b/cmd/derod/rpc_dero_gettransactions.go index ac7228b..bc3d8f8 100644 --- a/cmd/derod/rpc_dero_gettransactions.go +++ b/cmd/derod/rpc_dero_gettransactions.go @@ -94,20 +94,17 @@ func (DERO_RPC_APIS) GetTransaction(ctx context.Context, p rpc.GetTransaction_Pa } // also fill where the tx is found and in which block is valid and in which it is invalid - - valid_blid, invalid_blid, valid := chain.IS_TX_Valid(hash) + blid_list,state_block,state_block_topo := chain.IS_TX_Mined(hash) //logger.Infof(" tx %s related info valid_blid %s invalid_blid %+v valid %v ",hash, valid_blid, invalid_blid, valid) - if valid { - related.ValidBlock = valid_blid.String() - // topo height at which it was mined - topo_height := int64(chain.Load_Block_Topological_order(valid_blid)) - related.Block_Height = topo_height + if state_block_topo > 0 { + related.StateBlock = state_block.String() + related.Block_Height = state_block_topo if tx.TransactionType != transaction.REGISTRATION { // we must now fill in compressed ring members - if toporecord, err := chain.Store.Topo_store.Read(topo_height); err == nil { + if toporecord, err := chain.Store.Topo_store.Read(state_block_topo); err == nil { if ss, err := chain.Store.Balance_store.LoadSnapshot(toporecord.State_Version); err == nil { if tx.TransactionType == transaction.SC_TX { @@ -141,7 +138,6 @@ func (DERO_RPC_APIS) GetTransaction(ctx context.Context, p rpc.GetTransaction_Pa if tx.Payloads[t].SCID.IsZero() { tree, err = ss.GetTree(config.BALANCE_TREE) - } else { tree, err = ss.GetTree(string(tx.Payloads[t].SCID[:])) } @@ -166,8 +162,8 @@ func (DERO_RPC_APIS) GetTransaction(ctx context.Context, p rpc.GetTransaction_Pa } } } - for i := range invalid_blid { - related.InvalidBlock = append(related.InvalidBlock, invalid_blid[i].String()) + for i := range blid_list { + related.MinedBlock = append(related.MinedBlock, blid_list[i].String()) } result.Txs_as_hex = append(result.Txs_as_hex, hex.EncodeToString(tx.Serialize())) diff --git a/cmd/derod/rpc_dero_sendrawtransaction.go b/cmd/derod/rpc_dero_sendrawtransaction.go index c92b0f8..94ffc70 100644 --- a/cmd/derod/rpc_dero_sendrawtransaction.go +++ b/cmd/derod/rpc_dero_sendrawtransaction.go @@ -64,8 +64,8 @@ func (DERO_RPC_APIS) SendRawTransaction(ctx context.Context, p rpc.SendRawTransa result.Status = "OK" rlog.Debugf("Incoming TXID %s from RPC Server successfully accepted by MEMPOOL", tx.GetHash()) } else { - err = fmt.Errorf("Transaction %s rejected by daemon err '%s'", tx.GetHash(), err) - rlog.Warnf("Incoming TXID %s from RPC Server rejected by POOL", tx.GetHash()) + rlog.Warnf("Incoming TXID %s from RPC Server rejected by POOL err '%s'", tx.GetHash(),err) + err = fmt.Errorf("Transaction %s rejected by daemon err '%s'", tx.GetHash(), err) } return } diff --git a/cmd/derod/websocket_server.go b/cmd/derod/websocket_server.go index cbf0d0b..e00eb2e 100644 --- a/cmd/derod/websocket_server.go +++ b/cmd/derod/websocket_server.go @@ -26,11 +26,13 @@ import "sync/atomic" import "context" import "strings" import "runtime/debug" +import "net/http/pprof" import "github.com/romana/rlog" import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/globals" +import "github.com/deroproject/derohe/metrics" import "github.com/deroproject/derohe/blockchain" import "github.com/deroproject/derohe/glue/rwc" @@ -164,12 +166,15 @@ func (r *RPCServer) Run() { r.Unlock() r.mux.HandleFunc("/json_rpc", translate_http_to_jsonrpc_and_vice_versa) + r.mux.HandleFunc("/metrics", metrics.WritePrometheus) // write all the metrics + r.mux.HandleFunc("/ws", ws_handler) r.mux.HandleFunc("/", hello) - //r.mux.Handle("/json_rpc", mr) - - // handle nasty http requests - //r.mux.HandleFunc("/getheight", getheight) + r.mux.HandleFunc("/debug/pprof/", pprof.Index) + r.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + r.mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + r.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + r.mux.HandleFunc("/debug/pprof/trace", pprof.Trace) //if DEBUG_MODE { // r.mux.HandleFunc("/debug/pprof/", pprof.Index) @@ -182,21 +187,8 @@ func (r *RPCServer) Run() { r.mux.HandleFunc("/debug/pprof/trace", pprof.Trace) */ - /* - // Register pprof handlers individually if required - r.mux.HandleFunc("/cdebug/pprof/", pprof.Index) - r.mux.HandleFunc("/cdebug/pprof/cmdline", pprof.Cmdline) - r.mux.HandleFunc("/cdebug/pprof/profile", pprof.Profile) - r.mux.HandleFunc("/cdebug/pprof/symbol", pprof.Symbol) - r.mux.HandleFunc("/cdebug/pprof/trace", pprof.Trace) - */ - // register metrics handler - // r.mux.HandleFunc("/metrics", prometheus.InstrumentHandler("dero", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{}))) - //} - - //r.mux.HandleFunc("/json_rpc/debug", mr.ServeDebug) go Notify_Block_Addition() // process all blocks go Notify_Height_Changes() // gives notification of changed height diff --git a/cmd/explorer/explorer.go b/cmd/explorer/explorer.go index 8f3aa71..6592433 100644 --- a/cmd/explorer/explorer.go +++ b/cmd/explorer/explorer.go @@ -259,8 +259,8 @@ type txinfo struct { OutAddress []string // contains output secret key OutOffset []uint64 // contains index offsets Type string // ringct or ruffct ( bulletproof) - ValidBlock string // the tx is valid in which block - InvalidBlock []string // the tx is invalid in which block + StateBlock string // the tx is valid with reference to this block + MinedBlock []string // the tx is mined in which block Skipped bool // this is only valid, when a block is being listed Ring_size int Ring [][][]byte // contains entire ring in raw form @@ -399,9 +399,6 @@ func load_block_from_rpc(info *block_info, block_hash string, recursive bool) (e var tx txinfo err = load_tx_from_rpc(&tx, bl.Tx_hashes[i].String()) //TODO handle error fmt.Printf("loading tx %s err %s\n", bl.Tx_hashes[i].String(), err) - if tx.ValidBlock != bresult.Block_Header.Hash { // track skipped status - tx.Skipped = true - } info.Txs = append(info.Txs, tx) fees += tx.Feeuint64 size += tx.Sizeuint64 @@ -552,8 +549,8 @@ func load_tx_from_rpc(info *txinfo, txhash string) (err error) { info.Amount = fmt.Sprintf("%.05f", float64(uint64(tx_result.Txs[0].Reward))/100000) } - info.ValidBlock = tx_result.Txs[0].ValidBlock - info.InvalidBlock = tx_result.Txs[0].InvalidBlock + info.StateBlock = tx_result.Txs[0].StateBlock + info.MinedBlock = tx_result.Txs[0].MinedBlock info.Ring = tx_result.Txs[0].Ring diff --git a/cmd/explorer/templates.go b/cmd/explorer/templates.go index 276e05c..31fe679 100644 --- a/cmd/explorer/templates.go +++ b/cmd/explorer/templates.go @@ -308,9 +308,9 @@ var tx_template string = `{{define "tx"}}

Burns: {{.info.Burn_Value }} DERO

{{end}} -
Block: {{.info.ValidBlock}} (VALID)
+
Block: {{.info.StateBlock}} (Reference)
- {{range $i, $e := .info.InvalidBlock}} + {{range $i, $e := .info.MinedBlock}}
Block: {{$e}}
{{end}} diff --git a/config/config.go b/config/config.go index 4ab82cf..70f12f3 100644 --- a/config/config.go +++ b/config/config.go @@ -58,14 +58,16 @@ const MAINNET_MINIMUM_DIFFICULTY = uint64(800 * BLOCK_TIME) // 5 KH/s const TESTNET_BOOTSTRAP_DIFFICULTY = uint64(800 * BLOCK_TIME) // testnet bootstrap at 800 H/s const TESTNET_MINIMUM_DIFFICULTY = uint64(800 * BLOCK_TIME) // 800 H + +//this controls the batch size which controls till how many blocks incoming funds cannot be spend +const BLOCK_BATCH_SIZE = crypto.BLOCK_BATCH_SIZE + // this single parameter controls lots of various parameters // within the consensus, it should never go below 7 // if changed responsibly, we can have one second or lower blocks (ignoring chain bloat/size issues) // gives immense scalability, const STABLE_LIMIT = int64(8) -// reward percent that is shared between miners/dev -const DEVSHARE = uint64(600) // it's out of 10000, 600*100/10000 = 6%, 3% dev, 3% foundation // we can have number of chains running for testing reasons type CHAIN_CONFIG struct { @@ -104,7 +106,7 @@ var Mainnet = CHAIN_CONFIG{Name: "mainnet", } var Testnet = CHAIN_CONFIG{Name: "testnet", // testnet will always have last 3 bytes 0 - Network_ID: uuid.FromBytesOrNil([]byte{0x59, 0xd7, 0xf7, 0xe9, 0xdd, 0x48, 0xd5, 0xfd, 0x13, 0x0a, 0xf6, 0xe0, 0x26, 0x00, 0x00, 0x00}), + Network_ID: uuid.FromBytesOrNil([]byte{0x59, 0xd7, 0xf7, 0xe9, 0xdd, 0x48, 0xd5, 0xfd, 0x13, 0x0a, 0xf6, 0xe0, 0x26, 0x00, 0x02, 0x00}), P2P_Default_Port: 40401, RPC_Default_Port: 40402, Wallet_RPC_Default_Port: 40403, diff --git a/config/version.go b/config/version.go index 86af59b..37e4f8b 100644 --- a/config/version.go +++ b/config/version.go @@ -20,4 +20,4 @@ import "github.com/blang/semver" // right now it has to be manually changed // do we need to include git commitsha?? -var Version = semver.MustParse("3.2.14-1.DEROHE.STARGATE+28022021") +var Version = semver.MustParse("3.2.15-1.DEROHE.STARGATE+08082021") diff --git a/cryptography/crypto/proof_generate.go b/cryptography/crypto/proof_generate.go index e37d572..5f049d1 100644 --- a/cryptography/crypto/proof_generate.go +++ b/cryptography/crypto/proof_generate.go @@ -22,7 +22,7 @@ import "math/big" import "bytes" //import "crypto/rand" -//import "encoding/hex" +import "encoding/binary" import "github.com/deroproject/derohe/cryptography/bn256" @@ -30,6 +30,10 @@ import "github.com/deroproject/derohe/cryptography/bn256" //import "github.com/kubernetes/klog" + +// see comment in config package +const BLOCK_BATCH_SIZE = 10 + type Proof struct { BA *bn256.G1 BS *bn256.G1 @@ -39,6 +43,7 @@ type Proof struct { CLnG, CRnG, C_0G, DG, y_0G, gG, C_XG, y_XG []*bn256.G1 u *bn256.G1 + u1 *bn256.G1 f *FieldVector @@ -67,9 +72,12 @@ type IPWitness struct { } // this is based on roothash and user's secret key and thus is the basis of protection from a number of double spending attacks -func (p *Proof) Nonce() Hash { +func (p *Proof) Nonce1() Hash { return Keccak256(p.u.EncodeCompressed()) } +func (p *Proof) Nonce2() Hash { + return Keccak256(p.u1.EncodeCompressed()) +} func (p *Proof) Serialize(w *bytes.Buffer) { if p == nil { @@ -95,6 +103,7 @@ func (p *Proof) Serialize(w *bytes.Buffer) { } w.Write(p.u.EncodeCompressed()) + w.Write(p.u1.EncodeCompressed()) if len(p.CLnG) != len(p.f.vector) { /// panic(fmt.Sprintf("different size %d %d", len(p.CLnG), len(p.f.vector))) @@ -278,6 +287,16 @@ func (proof *Proof) Deserialize(r *bytes.Reader, length int) error { return err } + if n, err := r.Read(bufp[:]); n == 33 && err == nil { + var p bn256.G1 + if err = p.DecodeCompressed(bufp[:]); err != nil { + return err + } + proof.u1 = &p + } else { + return err + } + proof.f = &FieldVector{} //fmt.Printf("flen %d\n", flen ) @@ -438,12 +457,27 @@ func reverse(s string) string { return string(rns) } + +func HeightToPoint(height uint64) *bn256.G1 { + var input []byte + var h [8]byte + input = append(input, []byte(PROTOCOL_CONSTANT)...) + + binary.BigEndian.PutUint64(h[:], height) + input = append(input,h[:]...) + + + point := HashToPoint(HashtoNumber(input)) + return point +} + var params = NewGeneratorParams(128) // these can be pregenerated similarly as in DERO project -func GenerateProof(s *Statement, witness *Witness, u *bn256.G1, txid Hash, burn_value uint64) *Proof { +func GenerateProof(s *Statement, witness *Witness, u,u1 *bn256.G1, height uint64, txid Hash, burn_value uint64) *Proof { var proof Proof proof.u = u + proof.u1 = u1 statementhash := reducedhash(txid[:]) @@ -1106,17 +1140,9 @@ func GenerateProof(s *Statement, witness *Witness, u *bn256.G1, txid Hash, burn_ A_t := new(bn256.G1).ScalarMult(params.G, new(big.Int).Mod(new(big.Int).Neg(k_b), bn256.Order)) A_t = new(bn256.G1).Add(A_t, new(bn256.G1).ScalarMult(params.H, k_tau)) - A_u := new(bn256.G1) + A_u := new(bn256.G1).ScalarMult(HeightToPoint(height), k_sk) + A_u1 := new(bn256.G1).ScalarMult(HeightToPoint(height + BLOCK_BATCH_SIZE), k_sk) - { - var input []byte - input = append(input, []byte(PROTOCOL_CONSTANT)...) - input = append(input, s.Roothash[:]...) - - point := HashToPoint(HashtoNumber(input)) - - A_u = new(bn256.G1).ScalarMult(point, k_sk) - } // klog.V(2).Infof("A_y %s\n", A_y.String()) // klog.V(2).Infof("A_D %s\n", A_D.String()) @@ -1134,6 +1160,7 @@ func GenerateProof(s *Statement, witness *Witness, u *bn256.G1, txid Hash, burn_ input = append(input, A_X.Marshal()...) input = append(input, A_t.Marshal()...) input = append(input, A_u.Marshal()...) + input = append(input, A_u1.Marshal()...) proof.c = reducedhash(input) } diff --git a/cryptography/crypto/proof_verify.go b/cryptography/crypto/proof_verify.go index e4fd820..0c24d80 100644 --- a/cryptography/crypto/proof_verify.go +++ b/cryptography/crypto/proof_verify.go @@ -64,7 +64,7 @@ type ProtocolSupport struct { // sigma protocol type SigmaSupport struct { c *big.Int - A_y, A_D, A_b, A_X, A_t, A_u *bn256.G1 + A_y, A_D, A_b, A_X, A_t, A_u, A_u1 *bn256.G1 } // support structures are those which @@ -96,7 +96,7 @@ var gparams = NewGeneratorParams(128) // these can be pregenerated similarly as // verify proof // first generate supporting structures -func (proof *Proof) Verify(s *Statement, txid Hash, extra_value uint64) bool { +func (proof *Proof) Verify(s *Statement, txid Hash, height uint64, extra_value uint64) bool { var anonsupport AnonSupport var protsupport ProtocolSupport @@ -377,14 +377,14 @@ func (proof *Proof) Verify(s *Statement, txid Hash, extra_value uint64) bool { // klog.V(2).Infof("protsupport.tEval %s\n", protsupport.tEval.String()) { - var input []byte - input = append(input, []byte(PROTOCOL_CONSTANT)...) - input = append(input, s.Roothash[:]...) - - point := HashToPoint(HashtoNumber(input)) - + point := HeightToPoint(height) sigmasupport.A_u = new(bn256.G1).ScalarMult(point, proof.s_sk) sigmasupport.A_u.Add(new(bn256.G1).Set(sigmasupport.A_u), new(bn256.G1).ScalarMult(proof.u, proof_c_neg)) + + point = HeightToPoint(height+BLOCK_BATCH_SIZE) + sigmasupport.A_u1 = new(bn256.G1).ScalarMult(point, proof.s_sk) + sigmasupport.A_u1.Add(new(bn256.G1).Set(sigmasupport.A_u1), new(bn256.G1).ScalarMult(proof.u1, proof_c_neg)) + } // klog.V(2).Infof("A_y %s\n", sigmasupport.A_y.String()) @@ -403,6 +403,7 @@ func (proof *Proof) Verify(s *Statement, txid Hash, extra_value uint64) bool { input = append(input, sigmasupport.A_X.Marshal()...) input = append(input, sigmasupport.A_t.Marshal()...) input = append(input, sigmasupport.A_u.Marshal()...) + input = append(input, sigmasupport.A_u1.Marshal()...) // fmt.Printf("C calculation expected %s actual %s\n",proof.c.Text(16), reducedhash(input).Text(16) ) if reducedhash(input).Text(16) != proof.c.Text(16) { // we must fail here diff --git a/metrics/metrics.go b/metrics/metrics.go index baa269f..cc83412 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -18,14 +18,28 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import "net/http" +import "github.com/VictoriaMetrics/metrics" -var Registry = prometheus.NewRegistry() +// these are exported by the daemon for various analysis -var DefaultRegisterer prometheus.Registerer = Registry -var DefaultGatherer prometheus.Gatherer = Registry -// register some default go collectors -func init() { - Registry.MustRegister(prometheus.NewGoCollector()) -} +var Blockchain_tx_counter = metrics.NewCounter(`blockchain_tx_counter`) +var Mempool_tx_counter = metrics.NewCounter(`mempool_tx_counter`) +var Mempool_tx_count = metrics.NewCounter(`mempool_tx_count`) // its actually a gauge +var Block_size = metrics.NewHistogram(`block_size`) +var Block_tx = metrics.NewHistogram(`block_tx`) +var Block_processing_time = metrics.NewHistogram(`block_processing_time`) +var Transaction_size = metrics.NewHistogram(`transaction_size`) +var Transaction_ring_size = metrics.NewHistogram(`transaction_ring_size`) +var Transaction_outputs = metrics.NewHistogram("transaction_outputs") // a single tx will give to so many people + +// we may need to expose various p2p stats, but currently they can be skipped + +var Block_propagation = metrics.NewHistogram(`block_propagation`) +var Transaction_propagation = metrics.NewHistogram(`transaction_propagation`) + + +func WritePrometheus(w http.ResponseWriter, req *http.Request){ + metrics.WritePrometheus(w, true) +} \ No newline at end of file diff --git a/p2p/chain_bootstrap.go b/p2p/chain_bootstrap.go index 760865e..7685d72 100644 --- a/p2p/chain_bootstrap.go +++ b/p2p/chain_bootstrap.go @@ -56,7 +56,7 @@ func (connection *Connection) bootstrap_chain() { // we will request top 60 blocks ctopo := connection.TopoHeight var topos []int64 - for i := ctopo - 20; i < ctopo; i++ { + for i := ctopo - 200; i < ctopo; i++ { topos = append(topos, i) } diff --git a/p2p/connection_pool.go b/p2p/connection_pool.go index 9510401..bf10996 100644 --- a/p2p/connection_pool.go +++ b/p2p/connection_pool.go @@ -39,7 +39,6 @@ import "github.com/romana/rlog" import "github.com/dustin/go-humanize" import log "github.com/sirupsen/logrus" import "github.com/paulbellamy/ratecounter" -import "github.com/prometheus/client_golang/prometheus" import "github.com/deroproject/derohe/block" import "github.com/deroproject/derohe/cryptography/crypto" @@ -115,19 +114,6 @@ func (c *Connection) exit() { } -// 300 such buckets can be used to track block propagation accuratly upto a minute -var block_propagation = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "block_propagation_ms", - Help: "Block Propagation time milliseconds as detected by daemon", - Buckets: prometheus.LinearBuckets(0, 1000, 20), // start 0 ms, each 1000 ms, 20 such buckets. -}) - -// 300 such buckets can be used to track transaction propagation accurately upto a minute -var transaction_propagation = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "tx_propagation_ms", - Help: "TX Propagation time milliseconds as detected by daemon", - Buckets: prometheus.LinearBuckets(0, 1000, 20), // start 0 ms, each 1000 ms, 20 such buckets. -}) var block_propagation_map sync.Map var tx_propagation_map sync.Map diff --git a/p2p/controller.go b/p2p/controller.go index da0f54b..add4f3c 100644 --- a/p2p/controller.go +++ b/p2p/controller.go @@ -43,7 +43,7 @@ import log "github.com/sirupsen/logrus" import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/globals" import "github.com/deroproject/derohe/blockchain" -import "github.com/deroproject/derohe/metrics" + var chain *blockchain.Blockchain // external reference to chain @@ -105,9 +105,6 @@ func P2P_Init(params map[string]interface{}) error { } } - // register the metrics with the metrics registry - metrics.Registry.MustRegister(block_propagation) - metrics.Registry.MustRegister(transaction_propagation) go P2P_Server_v2() // start accepting connections go P2P_engine() // start outgoing engine diff --git a/p2p/rpc_notifications.go b/p2p/rpc_notifications.go index 6baff35..8af49e3 100644 --- a/p2p/rpc_notifications.go +++ b/p2p/rpc_notifications.go @@ -26,6 +26,7 @@ import "github.com/deroproject/derohe/globals" import "github.com/deroproject/derohe/block" import "github.com/deroproject/derohe/errormsg" import "github.com/deroproject/derohe/transaction" +import "github.com/deroproject/derohe/metrics" // notifies inventory func (c *Connection) NotifyINV(request ObjectList, response *Dummy) (err error) { @@ -104,9 +105,7 @@ func (c *Connection) NotifyTx(request Objects, response *Dummy) error { // track transaction propagation if first_time, ok := tx_propagation_map.Load(tx.GetHash()); ok { - // block already has a reference, take the time and observe the value - diff := time.Now().Sub(first_time.(time.Time)).Round(time.Millisecond) - transaction_propagation.Observe(float64(diff / 1000000)) + metrics.Transaction_propagation.UpdateDuration(first_time.(time.Time)) } else { tx_propagation_map.Store(tx.GetHash(), time.Now()) // if this is the first time, store the tx time } @@ -153,10 +152,8 @@ func (c *Connection) NotifyBlock(request Objects, response *Dummy) error { rlog.Infof("Incoming block Notification hash %s %s ", blid, globals.CTXString(c.logger)) // track block propagation - if first_time, ok := block_propagation_map.Load(blid); ok { - // block already has a reference, take the time and observe the value - diff := time.Now().Sub(first_time.(time.Time)).Round(time.Millisecond) - block_propagation.Observe(float64(diff / 1000000)) + if first_time, ok := block_propagation_map.Load(blid); ok { // block already has a reference, take the time and observe the value + metrics.Block_propagation.UpdateDuration(first_time.(time.Time)) } else { block_propagation_map.Store(blid, time.Now()) // if this is the first time, store the block } diff --git a/rpc/daemon_rpc.go b/rpc/daemon_rpc.go index 4a01b9d..899b7d3 100644 --- a/rpc/daemon_rpc.go +++ b/rpc/daemon_rpc.go @@ -123,6 +123,9 @@ type ( } ) +const RECENT_BLOCK = int64(-1) // will give most recent data +const RECENT_BATCH_BLOCK = int64(-2) // will give data from recent block batch for tx building + //get encrypted balance call type ( GetEncryptedBalance_Params struct { @@ -193,8 +196,8 @@ type ( In_pool bool `json:"in_pool"` Output_Indices []uint64 `json:"output_indices"` Tx_hash string `json:"tx_hash"` - ValidBlock string `json:"valid_block"` // TX is valid in this block - InvalidBlock []string `json:"invalid_block"` // TX is invalid in this block, 0 or more + StateBlock string `json:"state_block"` // TX is built in reference to this block + MinedBlock []string `json:"mined_block"` // TX is mined in this block, 1 or more Ring [][][]byte `json:"ring"` // ring members completed, since tx contains compressed Balance uint64 `json:"balance"` // if tx is SC, give SC balance at start Code string `json:"code"` // smart contract code at start diff --git a/transaction/transaction.go b/transaction/transaction.go index a685771..1bb427d 100644 --- a/transaction/transaction.go +++ b/transaction/transaction.go @@ -170,7 +170,7 @@ func (a *AssetPayload) UnmarshalProofs(r *bytes.Reader) (err error) { type Transaction struct { Transaction_Prefix // same as Transaction_Prefix - Payloads []AssetPayload // each transaction can have a number os payloads + Payloads []AssetPayload // each transaction can have a number of payloads } // this excludes the proof part, so it can pruned diff --git a/vendor/github.com/VictoriaMetrics/metrics/.github/workflows/main.yml b/vendor/github.com/VictoriaMetrics/metrics/.github/workflows/main.yml new file mode 100644 index 0000000..d8d8d12 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/.github/workflows/main.yml @@ -0,0 +1,33 @@ +name: main +on: + - push + - pull_request +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Setup Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + - name: Code checkout + uses: actions/checkout@v1 + - name: Test + run: | + go test -v ./... -coverprofile=coverage.txt -covermode=atomic + go test -v ./... -race + - name: Build + run: | + GOOS=linux go build + GOOS=darwin go build + GOOS=freebsd go build + GOOS=windows go build + GOARCH=386 go build + - name: Publish coverage + uses: codecov/codecov-action@v1.0.6 + with: + token: ${{secrets.CODECOV_TOKEN}} + file: ./coverage.txt + diff --git a/vendor/github.com/VictoriaMetrics/metrics/LICENSE b/vendor/github.com/VictoriaMetrics/metrics/LICENSE new file mode 100644 index 0000000..539b7a4 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 VictoriaMetrics + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/VictoriaMetrics/metrics/README.md b/vendor/github.com/VictoriaMetrics/metrics/README.md new file mode 100644 index 0000000..5eef96a --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/README.md @@ -0,0 +1,104 @@ +[![Build Status](https://github.com/VictoriaMetrics/metrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/metrics/actions) +[![GoDoc](https://godoc.org/github.com/VictoriaMetrics/metrics?status.svg)](http://godoc.org/github.com/VictoriaMetrics/metrics) +[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/metrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/metrics) +[![codecov](https://codecov.io/gh/VictoriaMetrics/metrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/metrics) + + +# metrics - lightweight package for exporting metrics in Prometheus format + + +### Features + +* Lightweight. Has minimal number of third-party dependencies and all these deps are small. + See [this article](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) for details. +* Easy to use. See the [API docs](http://godoc.org/github.com/VictoriaMetrics/metrics). +* Fast. +* Allows exporting distinct metric sets via distinct endpoints. See [Set](http://godoc.org/github.com/VictoriaMetrics/metrics#Set). +* Supports [easy-to-use histograms](http://godoc.org/github.com/VictoriaMetrics/metrics#Histogram), which just work without any tuning. + Read more about VictoriaMetrics histograms at [this article](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). + + +### Limitations + +* It doesn't implement advanced functionality from [github.com/prometheus/client_golang](https://godoc.org/github.com/prometheus/client_golang). + + +### Usage + +```go +import "github.com/VictoriaMetrics/metrics" + +// Register various time series. +// Time series name may contain labels in Prometheus format - see below. +var ( + // Register counter without labels. + requestsTotal = metrics.NewCounter("requests_total") + + // Register summary with a single label. + requestDuration = metrics.NewSummary(`requests_duration_seconds{path="/foobar/baz"}`) + + // Register gauge with two labels. + queueSize = metrics.NewGauge(`queue_size{queue="foobar",topic="baz"}`, func() float64 { + return float64(foobarQueue.Len()) + }) + + // Register histogram with a single label. + responseSize = metrics.NewHistogram(`response_size{path="/foo/bar"}`) +) + +// ... +func requestHandler() { + // Increment requestTotal counter. + requestsTotal.Inc() + + startTime := time.Now() + processRequest() + // Update requestDuration summary. + requestDuration.UpdateDuration(startTime) + + // Update responseSize histogram. + responseSize.Update(responseSize) +} + +// Expose the registered metrics at `/metrics` path. +http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics.WritePrometheus(w, true) +}) +``` + +See [docs](http://godoc.org/github.com/VictoriaMetrics/metrics) for more info. + + +### Users + +* `Metrics` has been extracted from [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) sources. + See [this article](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac) + for more info about `VictoriaMetrics`. + + +### FAQ + +#### Why the `metrics` API isn't compatible with `github.com/prometheus/client_golang`? + +Because the `github.com/prometheus/client_golang` is too complex and is hard to use. + + +#### Why the `metrics.WritePrometheus` doesn't expose documentation for each metric? + +Because this documentation is ignored by Prometheus. The documentation is for users. +Just give meaningful names to the exported metrics or add comments in the source code +or in other suitable place explaining each metric exposed from your application. + + +#### How to implement [CounterVec](https://godoc.org/github.com/prometheus/client_golang/prometheus#CounterVec) in `metrics`? + +Just use [GetOrCreateCounter](http://godoc.org/github.com/VictoriaMetrics/metrics#GetOrCreateCounter) +instead of `CounterVec.With`. See [this example](https://pkg.go.dev/github.com/VictoriaMetrics/metrics#example-Counter-Vec) for details. + + +#### Why [Histogram](http://godoc.org/github.com/VictoriaMetrics/metrics#Histogram) buckets contain `vmrange` labels instead of `le` labels like in Prometheus histograms? + +Buckets with `vmrange` labels occupy less disk space compared to Promethes-style buckets with `le` labels, +because `vmrange` buckets don't include counters for the previous ranges. [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) provides `prometheus_buckets` +function, which converts `vmrange` buckets to Prometheus-style buckets with `le` labels. This is useful for building heatmaps in Grafana. +Additionally, its' `histogram_quantile` function transparently handles histogram buckets with `vmrange` labels. diff --git a/vendor/github.com/VictoriaMetrics/metrics/counter.go b/vendor/github.com/VictoriaMetrics/metrics/counter.go new file mode 100644 index 0000000..a7d9549 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/counter.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "fmt" + "io" + "sync/atomic" +) + +// NewCounter registers and returns new counter with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +func NewCounter(name string) *Counter { + return defaultSet.NewCounter(name) +} + +// Counter is a counter. +// +// It may be used as a gauge if Dec and Set are called. +type Counter struct { + n uint64 +} + +// Inc increments c. +func (c *Counter) Inc() { + atomic.AddUint64(&c.n, 1) +} + +// Dec decrements c. +func (c *Counter) Dec() { + atomic.AddUint64(&c.n, ^uint64(0)) +} + +// Add adds n to c. +func (c *Counter) Add(n int) { + atomic.AddUint64(&c.n, uint64(n)) +} + +// Get returns the current value for c. +func (c *Counter) Get() uint64 { + return atomic.LoadUint64(&c.n) +} + +// Set sets c value to n. +func (c *Counter) Set(n uint64) { + atomic.StoreUint64(&c.n, n) +} + +// marshalTo marshals c with the given prefix to w. +func (c *Counter) marshalTo(prefix string, w io.Writer) { + v := c.Get() + fmt.Fprintf(w, "%s %d\n", prefix, v) +} + +// GetOrCreateCounter returns registered counter with the given name +// or creates new counter if the registry doesn't contain counter with +// the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewCounter instead of GetOrCreateCounter. +func GetOrCreateCounter(name string) *Counter { + return defaultSet.GetOrCreateCounter(name) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/counter_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/counter_example_test.go new file mode 100644 index 0000000..c485df6 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/counter_example_test.go @@ -0,0 +1,41 @@ +package metrics_test + +import ( + "fmt" + "github.com/VictoriaMetrics/metrics" +) + +func ExampleCounter() { + // Define a counter in global scope. + var c = metrics.NewCounter(`metric_total{label1="value1", label2="value2"}`) + + // Increment the counter when needed. + for i := 0; i < 10; i++ { + c.Inc() + } + n := c.Get() + fmt.Println(n) + + // Output: + // 10 +} + +func ExampleCounter_vec() { + for i := 0; i < 3; i++ { + // Dynamically construct metric name and pass it to GetOrCreateCounter. + name := fmt.Sprintf(`metric_total{label1=%q, label2="%d"}`, "value1", i) + metrics.GetOrCreateCounter(name).Add(i + 1) + } + + // Read counter values. + for i := 0; i < 3; i++ { + name := fmt.Sprintf(`metric_total{label1=%q, label2="%d"}`, "value1", i) + n := metrics.GetOrCreateCounter(name).Get() + fmt.Println(n) + } + + // Output: + // 1 + // 2 + // 3 +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/counter_test.go b/vendor/github.com/VictoriaMetrics/metrics/counter_test.go new file mode 100644 index 0000000..27c9264 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/counter_test.go @@ -0,0 +1,76 @@ +package metrics + +import ( + "fmt" + "testing" +) + +func TestCounterSerial(t *testing.T) { + name := "CounterSerial" + c := NewCounter(name) + c.Inc() + if n := c.Get(); n != 1 { + t.Fatalf("unexpected counter value; got %d; want 1", n) + } + c.Set(123) + if n := c.Get(); n != 123 { + t.Fatalf("unexpected counter value; got %d; want 123", n) + } + c.Dec() + if n := c.Get(); n != 122 { + t.Fatalf("unexpected counter value; got %d; want 122", n) + } + c.Add(3) + if n := c.Get(); n != 125 { + t.Fatalf("unexpected counter value; got %d; want 125", n) + } + + // Verify MarshalTo + testMarshalTo(t, c, "foobar", "foobar 125\n") +} + +func TestCounterConcurrent(t *testing.T) { + name := "CounterConcurrent" + c := NewCounter(name) + err := testConcurrent(func() error { + nPrev := c.Get() + for i := 0; i < 10; i++ { + c.Inc() + if n := c.Get(); n <= nPrev { + return fmt.Errorf("counter value must be greater than %d; got %d", nPrev, n) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateCounterSerial(t *testing.T) { + name := "GetOrCreateCounterSerial" + if err := testGetOrCreateCounter(name); err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateCounterConcurrent(t *testing.T) { + name := "GetOrCreateCounterConcurrent" + err := testConcurrent(func() error { + return testGetOrCreateCounter(name) + }) + if err != nil { + t.Fatal(err) + } +} + +func testGetOrCreateCounter(name string) error { + c1 := GetOrCreateCounter(name) + for i := 0; i < 10; i++ { + c2 := GetOrCreateCounter(name) + if c1 != c2 { + return fmt.Errorf("unexpected counter returned; got %p; want %p", c2, c1) + } + } + return nil +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go b/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go new file mode 100644 index 0000000..d01dd85 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "fmt" + "io" + "sync" +) + +// NewFloatCounter registers and returns new counter of float64 type with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +func NewFloatCounter(name string) *FloatCounter { + return defaultSet.NewFloatCounter(name) +} + +// FloatCounter is a float64 counter guarded by RWmutex. +// +// It may be used as a gauge if Add and Sub are called. +type FloatCounter struct { + mu sync.Mutex + n float64 +} + +// Add adds n to fc. +func (fc *FloatCounter) Add(n float64) { + fc.mu.Lock() + fc.n += n + fc.mu.Unlock() +} + +// Sub substracts n from fc. +func (fc *FloatCounter) Sub(n float64) { + fc.mu.Lock() + fc.n -= n + fc.mu.Unlock() +} + +// Get returns the current value for fc. +func (fc *FloatCounter) Get() float64 { + fc.mu.Lock() + n := fc.n + fc.mu.Unlock() + return n +} + +// Set sets fc value to n. +func (fc *FloatCounter) Set(n float64) { + fc.mu.Lock() + fc.n = n + fc.mu.Unlock() +} + +// marshalTo marshals fc with the given prefix to w. +func (fc *FloatCounter) marshalTo(prefix string, w io.Writer) { + v := fc.Get() + fmt.Fprintf(w, "%s %g\n", prefix, v) +} + +// GetOrCreateFloatCounter returns registered FloatCounter with the given name +// or creates new FloatCounter if the registry doesn't contain FloatCounter with +// the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned FloatCounter is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewFloatCounter instead of GetOrCreateFloatCounter. +func GetOrCreateFloatCounter(name string) *FloatCounter { + return defaultSet.GetOrCreateFloatCounter(name) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/floatcounter_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/floatcounter_example_test.go new file mode 100644 index 0000000..3f32770 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/floatcounter_example_test.go @@ -0,0 +1,41 @@ +package metrics_test + +import ( + "fmt" + "github.com/VictoriaMetrics/metrics" +) + +func ExampleFloatCounter() { + // Define a float64 counter in global scope. + var fc = metrics.NewFloatCounter(`float_metric_total{label1="value1", label2="value2"}`) + + // Add to the counter when needed. + for i := 0; i < 10; i++ { + fc.Add(1.01) + } + n := fc.Get() + fmt.Println(n) + + // Output: + // 10.1 +} + +func ExampleFloatCounter_vec() { + for i := 0; i < 3; i++ { + // Dynamically construct metric name and pass it to GetOrCreateFloatCounter. + name := fmt.Sprintf(`float_metric_total{label1=%q, label2="%d"}`, "value1", i) + metrics.GetOrCreateFloatCounter(name).Add(float64(i) + 1.01) + } + + // Read counter values. + for i := 0; i < 3; i++ { + name := fmt.Sprintf(`float_metric_total{label1=%q, label2="%d"}`, "value1", i) + n := metrics.GetOrCreateFloatCounter(name).Get() + fmt.Println(n) + } + + // Output: + // 1.01 + // 2.01 + // 3.01 +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/floatcounter_test.go b/vendor/github.com/VictoriaMetrics/metrics/floatcounter_test.go new file mode 100644 index 0000000..44931c3 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/floatcounter_test.go @@ -0,0 +1,76 @@ +package metrics + +import ( + "fmt" + "testing" +) + +func TestFloatCounterSerial(t *testing.T) { + name := "FloatCounterSerial" + c := NewFloatCounter(name) + c.Add(0.1) + if n := c.Get(); n != 0.1 { + t.Fatalf("unexpected counter value; got %f; want 0.1", n) + } + c.Set(123.00001) + if n := c.Get(); n != 123.00001 { + t.Fatalf("unexpected counter value; got %f; want 123.00001", n) + } + c.Sub(0.00001) + if n := c.Get(); n != 123 { + t.Fatalf("unexpected counter value; got %f; want 123", n) + } + c.Add(2.002) + if n := c.Get(); n != 125.002 { + t.Fatalf("unexpected counter value; got %f; want 125.002", n) + } + + // Verify MarshalTo + testMarshalTo(t, c, "foobar", "foobar 125.002\n") +} + +func TestFloatCounterConcurrent(t *testing.T) { + name := "FloatCounterConcurrent" + c := NewFloatCounter(name) + err := testConcurrent(func() error { + nPrev := c.Get() + for i := 0; i < 10; i++ { + c.Add(1.001) + if n := c.Get(); n <= nPrev { + return fmt.Errorf("counter value must be greater than %f; got %f", nPrev, n) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateFloatCounterSerial(t *testing.T) { + name := "GetOrCreateFloatCounterSerial" + if err := testGetOrCreateCounter(name); err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateFloatCounterConcurrent(t *testing.T) { + name := "GetOrCreateFloatCounterConcurrent" + err := testConcurrent(func() error { + return testGetOrCreateFloatCounter(name) + }) + if err != nil { + t.Fatal(err) + } +} + +func testGetOrCreateFloatCounter(name string) error { + c1 := GetOrCreateFloatCounter(name) + for i := 0; i < 10; i++ { + c2 := GetOrCreateFloatCounter(name) + if c1 != c2 { + return fmt.Errorf("unexpected counter returned; got %p; want %p", c2, c1) + } + } + return nil +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/gauge.go b/vendor/github.com/VictoriaMetrics/metrics/gauge.go new file mode 100644 index 0000000..05bf147 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/gauge.go @@ -0,0 +1,67 @@ +package metrics + +import ( + "fmt" + "io" +) + +// NewGauge registers and returns gauge with the given name, which calls f +// to obtain gauge value. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// f must be safe for concurrent calls. +// +// The returned gauge is safe to use from concurrent goroutines. +// +// See also FloatCounter for working with floating-point values. +func NewGauge(name string, f func() float64) *Gauge { + return defaultSet.NewGauge(name, f) +} + +// Gauge is a float64 gauge. +// +// See also Counter, which could be used as a gauge with Set and Dec calls. +type Gauge struct { + f func() float64 +} + +// Get returns the current value for g. +func (g *Gauge) Get() float64 { + return g.f() +} + +func (g *Gauge) marshalTo(prefix string, w io.Writer) { + v := g.f() + if float64(int64(v)) == v { + // Marshal integer values without scientific notation + fmt.Fprintf(w, "%s %d\n", prefix, int64(v)) + } else { + fmt.Fprintf(w, "%s %g\n", prefix, v) + } +} + +// GetOrCreateGauge returns registered gauge with the given name +// or creates new gauge if the registry doesn't contain gauge with +// the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned gauge is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewGauge instead of GetOrCreateGauge. +// +// See also FloatCounter for working with floating-point values. +func GetOrCreateGauge(name string, f func() float64) *Gauge { + return defaultSet.GetOrCreateGauge(name, f) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/gauge_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/gauge_example_test.go new file mode 100644 index 0000000..0f0e440 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/gauge_example_test.go @@ -0,0 +1,41 @@ +package metrics_test + +import ( + "fmt" + "runtime" + + "github.com/VictoriaMetrics/metrics" +) + +func ExampleGauge() { + // Define a gauge exporting the number of goroutines. + var g = metrics.NewGauge(`goroutines_count`, func() float64 { + return float64(runtime.NumGoroutine()) + }) + + // Obtain gauge value. + fmt.Println(g.Get()) +} + +func ExampleGauge_vec() { + for i := 0; i < 3; i++ { + // Dynamically construct metric name and pass it to GetOrCreateGauge. + name := fmt.Sprintf(`metric{label1=%q, label2="%d"}`, "value1", i) + iLocal := i + metrics.GetOrCreateGauge(name, func() float64 { + return float64(iLocal + 1) + }) + } + + // Read counter values. + for i := 0; i < 3; i++ { + name := fmt.Sprintf(`metric{label1=%q, label2="%d"}`, "value1", i) + n := metrics.GetOrCreateGauge(name, func() float64 { return 0 }).Get() + fmt.Println(n) + } + + // Output: + // 1 + // 2 + // 3 +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/gauge_test.go b/vendor/github.com/VictoriaMetrics/metrics/gauge_test.go new file mode 100644 index 0000000..3f6e5cd --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/gauge_test.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "sync" + "testing" +) + +func TestGaugeError(t *testing.T) { + expectPanic(t, "NewGauge_nil_callback", func() { + NewGauge("NewGauge_nil_callback", nil) + }) + expectPanic(t, "GetOrCreateGauge_nil_callback", func() { + GetOrCreateGauge("GetOrCreateGauge_nil_callback", nil) + }) +} + +func TestGaugeSerial(t *testing.T) { + name := "GaugeSerial" + n := 1.23 + var nLock sync.Mutex + g := NewGauge(name, func() float64 { + nLock.Lock() + defer nLock.Unlock() + n++ + return n + }) + for i := 0; i < 10; i++ { + if nn := g.Get(); nn != n { + t.Fatalf("unexpected gauge value; got %v; want %v", nn, n) + } + } + + // Verify marshalTo + testMarshalTo(t, g, "foobar", "foobar 12.23\n") + + // Verify big numbers marshaling + n = 1234567899 + testMarshalTo(t, g, "prefix", "prefix 1234567900\n") +} + +func TestGaugeConcurrent(t *testing.T) { + name := "GaugeConcurrent" + var n int + var nLock sync.Mutex + g := NewGauge(name, func() float64 { + nLock.Lock() + defer nLock.Unlock() + n++ + return float64(n) + }) + err := testConcurrent(func() error { + nPrev := g.Get() + for i := 0; i < 10; i++ { + if n := g.Get(); n <= nPrev { + return fmt.Errorf("gauge value must be greater than %v; got %v", nPrev, n) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/go.mod b/vendor/github.com/VictoriaMetrics/metrics/go.mod new file mode 100644 index 0000000..a66c19b --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/go.mod @@ -0,0 +1,5 @@ +module github.com/VictoriaMetrics/metrics + +require github.com/valyala/histogram v1.1.2 + +go 1.12 diff --git a/vendor/github.com/VictoriaMetrics/metrics/go.sum b/vendor/github.com/VictoriaMetrics/metrics/go.sum new file mode 100644 index 0000000..b121944 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/go.sum @@ -0,0 +1,4 @@ +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8= +github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg= diff --git a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go new file mode 100644 index 0000000..f8b6067 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "io" + "runtime" + + "github.com/valyala/histogram" +) + +func writeGoMetrics(w io.Writer) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc) + fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc) + fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys) + fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees) + fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction) + fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys) + fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc) + fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle) + fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse) + fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects) + fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased) + fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys) + fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9) + fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups) + fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs) + fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse) + fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys) + fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse) + fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys) + fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC) + fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys) + fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse) + fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys) + fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys) + + fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall()) + fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU()) + + gcPauses := histogram.NewFast() + for _, pauseNs := range ms.PauseNs[:] { + gcPauses.Update(float64(pauseNs) / 1e9) + } + phis := []float64{0, 0.25, 0.5, 0.75, 1} + quantiles := make([]float64, 0, len(phis)) + for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { + fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q) + } + fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) + fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + + fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) + fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + numThread, _ := runtime.ThreadCreateProfile(nil) + fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + + // Export build details. + fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", + runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/histogram.go b/vendor/github.com/VictoriaMetrics/metrics/histogram.go new file mode 100644 index 0000000..b0e8d57 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/histogram.go @@ -0,0 +1,230 @@ +package metrics + +import ( + "fmt" + "io" + "math" + "sync" + "time" +) + +const ( + e10Min = -9 + e10Max = 18 + bucketsPerDecimal = 18 + decimalBucketsCount = e10Max - e10Min + bucketsCount = decimalBucketsCount * bucketsPerDecimal +) + +var bucketMultiplier = math.Pow(10, 1.0/bucketsPerDecimal) + +// Histogram is a histogram for non-negative values with automatically created buckets. +// +// See https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350 +// +// Each bucket contains a counter for values in the given range. +// Each non-empty bucket is exposed via the following metric: +// +// _bucket{,vmrange="..."} +// +// Where: +// +// - is the metric name passed to NewHistogram +// - is optional tags for the , which are passed to NewHistogram +// - and - start and end values for the given bucket +// - - the number of hits to the given bucket during Update* calls +// +// Histogram buckets can be converted to Prometheus-like buckets with `le` labels +// with `prometheus_buckets(_bucket)` function from PromQL extensions in VictoriaMetrics. +// (see https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL ): +// +// prometheus_buckets(request_duration_bucket) +// +// Time series produced by the Histogram have better compression ratio comparing to +// Prometheus histogram buckets with `le` labels, since they don't include counters +// for all the previous buckets. +// +// Zero histogram is usable. +type Histogram struct { + // Mu gurantees synchronous update for all the counters and sum. + mu sync.Mutex + + decimalBuckets [decimalBucketsCount]*[bucketsPerDecimal]uint64 + + lower uint64 + upper uint64 + + sum float64 +} + +// Reset resets the given histogram. +func (h *Histogram) Reset() { + h.mu.Lock() + for _, db := range h.decimalBuckets[:] { + if db == nil { + continue + } + for i := range db[:] { + db[i] = 0 + } + } + h.lower = 0 + h.upper = 0 + h.sum = 0 + h.mu.Unlock() +} + +// Update updates h with v. +// +// Negative values and NaNs are ignored. +func (h *Histogram) Update(v float64) { + if math.IsNaN(v) || v < 0 { + // Skip NaNs and negative values. + return + } + bucketIdx := (math.Log10(v) - e10Min) * bucketsPerDecimal + h.mu.Lock() + h.sum += v + if bucketIdx < 0 { + h.lower++ + } else if bucketIdx >= bucketsCount { + h.upper++ + } else { + idx := uint(bucketIdx) + if bucketIdx == float64(idx) && idx > 0 { + // Edge case for 10^n values, which must go to the lower bucket + // according to Prometheus logic for `le`-based histograms. + idx-- + } + decimalBucketIdx := idx / bucketsPerDecimal + offset := idx % bucketsPerDecimal + db := h.decimalBuckets[decimalBucketIdx] + if db == nil { + var b [bucketsPerDecimal]uint64 + db = &b + h.decimalBuckets[decimalBucketIdx] = db + } + db[offset]++ + } + h.mu.Unlock() +} + +// VisitNonZeroBuckets calls f for all buckets with non-zero counters. +// +// vmrange contains "..." string with bucket bounds. The lower bound +// isn't included in the bucket, while the upper bound is included. +// This is required to be compatible with Prometheus-style histogram buckets +// with `le` (less or equal) labels. +func (h *Histogram) VisitNonZeroBuckets(f func(vmrange string, count uint64)) { + h.mu.Lock() + if h.lower > 0 { + f(lowerBucketRange, h.lower) + } + for decimalBucketIdx, db := range h.decimalBuckets[:] { + if db == nil { + continue + } + for offset, count := range db[:] { + if count > 0 { + bucketIdx := decimalBucketIdx*bucketsPerDecimal + offset + vmrange := getVMRange(bucketIdx) + f(vmrange, count) + } + } + } + if h.upper > 0 { + f(upperBucketRange, h.upper) + } + h.mu.Unlock() +} + +// NewHistogram creates and returns new histogram with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned histogram is safe to use from concurrent goroutines. +func NewHistogram(name string) *Histogram { + return defaultSet.NewHistogram(name) +} + +// GetOrCreateHistogram returns registered histogram with the given name +// or creates new histogram if the registry doesn't contain histogram with +// the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned histogram is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewHistogram instead of GetOrCreateHistogram. +func GetOrCreateHistogram(name string) *Histogram { + return defaultSet.GetOrCreateHistogram(name) +} + +// UpdateDuration updates request duration based on the given startTime. +func (h *Histogram) UpdateDuration(startTime time.Time) { + d := time.Since(startTime).Seconds() + h.Update(d) +} + +func getVMRange(bucketIdx int) string { + bucketRangesOnce.Do(initBucketRanges) + return bucketRanges[bucketIdx] +} + +func initBucketRanges() { + v := math.Pow10(e10Min) + start := fmt.Sprintf("%.3e", v) + for i := 0; i < bucketsCount; i++ { + v *= bucketMultiplier + end := fmt.Sprintf("%.3e", v) + bucketRanges[i] = start + "..." + end + start = end + } +} + +var ( + lowerBucketRange = fmt.Sprintf("0...%.3e", math.Pow10(e10Min)) + upperBucketRange = fmt.Sprintf("%.3e...+Inf", math.Pow10(e10Max)) + + bucketRanges [bucketsCount]string + bucketRangesOnce sync.Once +) + +func (h *Histogram) marshalTo(prefix string, w io.Writer) { + countTotal := uint64(0) + h.VisitNonZeroBuckets(func(vmrange string, count uint64) { + tag := fmt.Sprintf("vmrange=%q", vmrange) + metricName := addTag(prefix, tag) + name, labels := splitMetricName(metricName) + fmt.Fprintf(w, "%s_bucket%s %d\n", name, labels, count) + countTotal += count + }) + if countTotal == 0 { + return + } + name, labels := splitMetricName(prefix) + sum := h.getSum() + if float64(int64(sum)) == sum { + fmt.Fprintf(w, "%s_sum%s %d\n", name, labels, int64(sum)) + } else { + fmt.Fprintf(w, "%s_sum%s %g\n", name, labels, sum) + } + fmt.Fprintf(w, "%s_count%s %d\n", name, labels, countTotal) +} + +func (h *Histogram) getSum() float64 { + h.mu.Lock() + sum := h.sum + h.mu.Unlock() + return sum +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/histogram_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/histogram_example_test.go new file mode 100644 index 0000000..358050e --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/histogram_example_test.go @@ -0,0 +1,27 @@ +package metrics_test + +import ( + "fmt" + "time" + + "github.com/VictoriaMetrics/metrics" +) + +func ExampleHistogram() { + // Define a histogram in global scope. + var h = metrics.NewHistogram(`request_duration_seconds{path="/foo/bar"}`) + + // Update the histogram with the duration of processRequest call. + startTime := time.Now() + processRequest() + h.UpdateDuration(startTime) +} + +func ExampleHistogram_vec() { + for i := 0; i < 3; i++ { + // Dynamically construct metric name and pass it to GetOrCreateHistogram. + name := fmt.Sprintf(`response_size_bytes{path=%q}`, "/foo/bar") + response := processRequest() + metrics.GetOrCreateHistogram(name).Update(float64(len(response))) + } +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/histogram_test.go b/vendor/github.com/VictoriaMetrics/metrics/histogram_test.go new file mode 100644 index 0000000..dea989b --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/histogram_test.go @@ -0,0 +1,200 @@ +package metrics + +import ( + "bytes" + "fmt" + "math" + "reflect" + "strings" + "testing" + "time" +) + +func TestGetVMRange(t *testing.T) { + f := func(bucketIdx int, vmrangeExpected string) { + t.Helper() + vmrange := getVMRange(bucketIdx) + if vmrange != vmrangeExpected { + t.Fatalf("unexpected vmrange for bucketIdx=%d; got %s; want %s", bucketIdx, vmrange, vmrangeExpected) + } + } + f(0, "1.000e-09...1.136e-09") + f(1, "1.136e-09...1.292e-09") + f(bucketsPerDecimal-1, "8.799e-09...1.000e-08") + f(bucketsPerDecimal, "1.000e-08...1.136e-08") + f(bucketsPerDecimal*(-e10Min)-1, "8.799e-01...1.000e+00") + f(bucketsPerDecimal*(-e10Min), "1.000e+00...1.136e+00") + f(bucketsPerDecimal*(e10Max-e10Min)-1, "8.799e+17...1.000e+18") +} + +func TestHistogramSerial(t *testing.T) { + name := `TestHistogramSerial` + h := NewHistogram(name) + + // Verify that the histogram is invisible in the output of WritePrometheus when it has no data. + var bb bytes.Buffer + WritePrometheus(&bb, false) + result := bb.String() + if strings.Contains(result, name) { + t.Fatalf("histogram %s shouldn't be visible in the WritePrometheus output; got\n%s", name, result) + } + + // Write data to histogram + for i := 98; i < 218; i++ { + h.Update(float64(i)) + } + + // Make sure the histogram prints _bucket on marshalTo call + testMarshalTo(t, h, "prefix", `prefix_bucket{vmrange="8.799e+01...1.000e+02"} 3 +prefix_bucket{vmrange="1.000e+02...1.136e+02"} 13 +prefix_bucket{vmrange="1.136e+02...1.292e+02"} 16 +prefix_bucket{vmrange="1.292e+02...1.468e+02"} 17 +prefix_bucket{vmrange="1.468e+02...1.668e+02"} 20 +prefix_bucket{vmrange="1.668e+02...1.896e+02"} 23 +prefix_bucket{vmrange="1.896e+02...2.154e+02"} 26 +prefix_bucket{vmrange="2.154e+02...2.448e+02"} 2 +prefix_sum 18900 +prefix_count 120 +`) + testMarshalTo(t, h, ` m{foo="bar"}`, ` m_bucket{foo="bar",vmrange="8.799e+01...1.000e+02"} 3 + m_bucket{foo="bar",vmrange="1.000e+02...1.136e+02"} 13 + m_bucket{foo="bar",vmrange="1.136e+02...1.292e+02"} 16 + m_bucket{foo="bar",vmrange="1.292e+02...1.468e+02"} 17 + m_bucket{foo="bar",vmrange="1.468e+02...1.668e+02"} 20 + m_bucket{foo="bar",vmrange="1.668e+02...1.896e+02"} 23 + m_bucket{foo="bar",vmrange="1.896e+02...2.154e+02"} 26 + m_bucket{foo="bar",vmrange="2.154e+02...2.448e+02"} 2 + m_sum{foo="bar"} 18900 + m_count{foo="bar"} 120 +`) + + // Verify Reset + h.Reset() + bb.Reset() + WritePrometheus(&bb, false) + result = bb.String() + if strings.Contains(result, name) { + t.Fatalf("unexpected histogram %s in the WritePrometheus output; got\n%s", name, result) + } + + // Verify supported ranges + for e10 := -100; e10 < 100; e10++ { + for offset := 0; offset < bucketsPerDecimal; offset++ { + m := 1 + math.Pow(bucketMultiplier, float64(offset)) + f1 := m * math.Pow10(e10) + h.Update(f1) + f2 := (m + 0.5*bucketMultiplier) * math.Pow10(e10) + h.Update(f2) + f3 := (m + 2*bucketMultiplier) * math.Pow10(e10) + h.Update(f3) + } + } + h.UpdateDuration(time.Now().Add(-time.Minute)) + + // Verify edge cases + h.Update(0) + h.Update(math.Inf(1)) + h.Update(math.Inf(-1)) + h.Update(math.NaN()) + h.Update(-123) + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1096 + h.Update(math.Float64frombits(0x3e112e0be826d695)) + + // Make sure the histogram becomes visible in the output of WritePrometheus, + // since now it contains values. + bb.Reset() + WritePrometheus(&bb, false) + result = bb.String() + if !strings.Contains(result, name) { + t.Fatalf("missing histogram %s in the WritePrometheus output; got\n%s", name, result) + } +} + +func TestHistogramConcurrent(t *testing.T) { + name := "HistogramConcurrent" + h := NewHistogram(name) + err := testConcurrent(func() error { + for f := 0.6; f < 1.4; f += 0.1 { + h.Update(f) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + testMarshalTo(t, h, "prefix", `prefix_bucket{vmrange="5.995e-01...6.813e-01"} 5 +prefix_bucket{vmrange="6.813e-01...7.743e-01"} 5 +prefix_bucket{vmrange="7.743e-01...8.799e-01"} 5 +prefix_bucket{vmrange="8.799e-01...1.000e+00"} 10 +prefix_bucket{vmrange="1.000e+00...1.136e+00"} 5 +prefix_bucket{vmrange="1.136e+00...1.292e+00"} 5 +prefix_bucket{vmrange="1.292e+00...1.468e+00"} 5 +prefix_sum 38 +prefix_count 40 +`) + + var labels []string + var counts []uint64 + h.VisitNonZeroBuckets(func(label string, count uint64) { + labels = append(labels, label) + counts = append(counts, count) + }) + labelsExpected := []string{ + "5.995e-01...6.813e-01", + "6.813e-01...7.743e-01", + "7.743e-01...8.799e-01", + "8.799e-01...1.000e+00", + "1.000e+00...1.136e+00", + "1.136e+00...1.292e+00", + "1.292e+00...1.468e+00", + } + if !reflect.DeepEqual(labels, labelsExpected) { + t.Fatalf("unexpected labels; got %v; want %v", labels, labelsExpected) + } + countsExpected := []uint64{5, 5, 5, 10, 5, 5, 5} + if !reflect.DeepEqual(counts, countsExpected) { + t.Fatalf("unexpected counts; got %v; want %v", counts, countsExpected) + } +} + +func TestHistogramWithTags(t *testing.T) { + name := `TestHistogram{tag="foo"}` + h := NewHistogram(name) + h.Update(123) + + var bb bytes.Buffer + WritePrometheus(&bb, false) + result := bb.String() + namePrefixWithTag := `TestHistogram_bucket{tag="foo",vmrange="1.136e+02...1.292e+02"} 1` + "\n" + if !strings.Contains(result, namePrefixWithTag) { + t.Fatalf("missing histogram %s in the WritePrometheus output; got\n%s", namePrefixWithTag, result) + } +} + +func TestGetOrCreateHistogramSerial(t *testing.T) { + name := "GetOrCreateHistogramSerial" + if err := testGetOrCreateHistogram(name); err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateHistogramConcurrent(t *testing.T) { + name := "GetOrCreateHistogramConcurrent" + err := testConcurrent(func() error { + return testGetOrCreateHistogram(name) + }) + if err != nil { + t.Fatal(err) + } +} + +func testGetOrCreateHistogram(name string) error { + h1 := GetOrCreateHistogram(name) + for i := 0; i < 10; i++ { + h2 := GetOrCreateHistogram(name) + if h1 != h2 { + return fmt.Errorf("unexpected histogram returned; got %p; want %p", h2, h1) + } + } + return nil +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/histogram_timing_test.go b/vendor/github.com/VictoriaMetrics/metrics/histogram_timing_test.go new file mode 100644 index 0000000..cee323f --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/histogram_timing_test.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "testing" +) + +func BenchmarkHistogramUpdate(b *testing.B) { + h := GetOrCreateHistogram("BenchmarkHistogramUpdate") + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + h.Update(float64(i)) + i++ + } + }) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go new file mode 100644 index 0000000..c28c036 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -0,0 +1,112 @@ +// Package metrics implements Prometheus-compatible metrics for applications. +// +// This package is lightweight alternative to https://github.com/prometheus/client_golang +// with simpler API and smaller dependencies. +// +// Usage: +// +// 1. Register the required metrics via New* functions. +// 2. Expose them to `/metrics` page via WritePrometheus. +// 3. Update the registered metrics during application lifetime. +// +// The package has been extracted from https://victoriametrics.com/ +package metrics + +import ( + "io" +) + +type namedMetric struct { + name string + metric metric +} + +type metric interface { + marshalTo(prefix string, w io.Writer) +} + +var defaultSet = NewSet() + +// WritePrometheus writes all the registered metrics in Prometheus format to w. +// +// If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics +// are exposed for the current process. +// +// The WritePrometheus func is usually called inside "/metrics" handler: +// +// http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { +// metrics.WritePrometheus(w, true) +// }) +// +func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { + defaultSet.WritePrometheus(w) + if exposeProcessMetrics { + WriteProcessMetrics(w) + } +} + +// WriteProcessMetrics writes additional process metrics in Prometheus format to w. +// +// The following `go_*` and `process_*` metrics are exposed for the currently +// running process. Below is a short description for the exposed `process_*` metrics: +// +// - process_cpu_seconds_system_total - CPU time spent in syscalls +// - process_cpu_seconds_user_total - CPU time spent in userspace +// - process_cpu_seconds_total - CPU time spent by the process +// - process_major_pagefaults_total - page faults resulted in disk IO +// - process_minor_pagefaults_total - page faults resolved without disk IO +// - process_resident_memory_bytes - recently accessed memory (aka RSS or resident memory) +// - process_resident_memory_peak_bytes - the maximum RSS memory usage +// - process_resident_memory_anon_bytes - RSS for memory-mapped files +// - process_resident_memory_file_bytes - RSS for memory allocated by the process +// - process_resident_memory_shared_bytes - RSS for memory shared between multiple processes +// - process_virtual_memory_bytes - virtual memory usage +// - process_virtual_memory_peak_bytes - the maximum virtual memory usage +// - process_num_threads - the number of threads +// - process_start_time_seconds - process start time as unix timestamp +// +// - process_io_read_bytes_total - the number of bytes read via syscalls +// - process_io_written_bytes_total - the number of bytes written via syscalls +// - process_io_read_syscalls_total - the number of read syscalls +// - process_io_write_syscalls_total - the number of write syscalls +// - process_io_storage_read_bytes_total - the number of bytes actually read from disk +// - process_io_storage_written_bytes_total - the number of bytes actually written to disk +// +// - go_memstats_alloc_bytes - memory usage for Go objects in the heap +// - go_memstats_alloc_bytes_total - the cumulative counter for total size of allocated Go objects +// - go_memstats_frees_total - the cumulative counter for number of freed Go objects +// - go_memstats_gc_cpu_fraction - the fraction of CPU spent in Go garbage collector +// - go_memstats_gc_sys_bytes - the size of Go garbage collector metadata +// - go_memstats_heap_alloc_bytes - the same as go_memstats_alloc_bytes +// - go_memstats_heap_idle_bytes - idle memory ready for new Go object allocations +// - go_memstats_heap_objects - the number of Go objects in the heap +// - go_memstats_heap_sys_bytes - memory requested for Go objects from the OS +// - go_memstats_mallocs_total - the number of allocations for Go objects +// - go_memstats_next_gc_bytes - the target heap size when the next garbage collection should start +// - go_memstats_stack_inuse_bytes - memory used for goroutine stacks +// - go_memstats_stack_sys_bytes - memory requested fromthe OS for goroutine stacks +// - go_memstats_sys_bytes - memory requested by Go runtime from the OS +// +// The WriteProcessMetrics func is usually called in combination with writing Set metrics +// inside "/metrics" handler: +// +// http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { +// mySet.WritePrometheus(w) +// metrics.WriteProcessMetrics(w) +// }) +// +// See also WrteFDMetrics. +func WriteProcessMetrics(w io.Writer) { + writeGoMetrics(w) + writeProcessMetrics(w) +} + +// WriteFDMetrics writes `process_max_fds` and `process_open_fds` metrics to w. +func WriteFDMetrics(w io.Writer) { + writeFDMetrics(w) +} + +// UnregisterMetric removes metric with the given name from default set. +func UnregisterMetric(name string) bool { + return defaultSet.UnregisterMetric(name) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/metrics_example_test.go new file mode 100644 index 0000000..0ac08ae --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics_example_test.go @@ -0,0 +1,14 @@ +package metrics_test + +import ( + "net/http" + + "github.com/VictoriaMetrics/metrics" +) + +func ExampleWritePrometheus() { + // Export all the registered metrics in Prometheus format at `/metrics` http path. + http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics.WritePrometheus(w, true) + }) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics_test.go b/vendor/github.com/VictoriaMetrics/metrics/metrics_test.go new file mode 100644 index 0000000..baa624c --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics_test.go @@ -0,0 +1,146 @@ +package metrics + +import ( + "bytes" + "fmt" + "testing" + "time" +) + +func TestInvalidName(t *testing.T) { + f := func(name string) { + t.Helper() + expectPanic(t, fmt.Sprintf("NewCounter(%q)", name), func() { NewCounter(name) }) + expectPanic(t, fmt.Sprintf("NewGauge(%q)", name), func() { NewGauge(name, func() float64 { return 0 }) }) + expectPanic(t, fmt.Sprintf("NewSummary(%q)", name), func() { NewSummary(name) }) + expectPanic(t, fmt.Sprintf("GetOrCreateCounter(%q)", name), func() { GetOrCreateCounter(name) }) + expectPanic(t, fmt.Sprintf("GetOrCreateGauge(%q)", name), func() { GetOrCreateGauge(name, func() float64 { return 0 }) }) + expectPanic(t, fmt.Sprintf("GetOrCreateSummary(%q)", name), func() { GetOrCreateSummary(name) }) + expectPanic(t, fmt.Sprintf("GetOrCreateHistogram(%q)", name), func() { GetOrCreateHistogram(name) }) + } + f("") + f("foo{") + f("foo}") + f("foo{bar") + f("foo{bar=") + f(`foo{bar="`) + f(`foo{bar="baz`) + f(`foo{bar="baz"`) + f(`foo{bar="baz",`) + f(`foo{bar="baz",}`) +} + +func TestDoubleRegister(t *testing.T) { + t.Run("NewCounter", func(t *testing.T) { + name := "NewCounterDoubleRegister" + NewCounter(name) + expectPanic(t, name, func() { NewCounter(name) }) + }) + t.Run("NewGauge", func(t *testing.T) { + name := "NewGaugeDoubleRegister" + NewGauge(name, func() float64 { return 0 }) + expectPanic(t, name, func() { NewGauge(name, func() float64 { return 0 }) }) + }) + t.Run("NewSummary", func(t *testing.T) { + name := "NewSummaryDoubleRegister" + NewSummary(name) + expectPanic(t, name, func() { NewSummary(name) }) + }) + t.Run("NewHistogram", func(t *testing.T) { + name := "NewHistogramDoubleRegister" + NewHistogram(name) + expectPanic(t, name, func() { NewSummary(name) }) + }) +} + +func TestGetOrCreateNotCounter(t *testing.T) { + name := "GetOrCreateNotCounter" + NewSummary(name) + expectPanic(t, name, func() { GetOrCreateCounter(name) }) +} + +func TestGetOrCreateNotGauge(t *testing.T) { + name := "GetOrCreateNotGauge" + NewCounter(name) + expectPanic(t, name, func() { GetOrCreateGauge(name, func() float64 { return 0 }) }) +} + +func TestGetOrCreateNotSummary(t *testing.T) { + name := "GetOrCreateNotSummary" + NewCounter(name) + expectPanic(t, name, func() { GetOrCreateSummary(name) }) +} + +func TestGetOrCreateNotHistogram(t *testing.T) { + name := "GetOrCreateNotHistogram" + NewCounter(name) + expectPanic(t, name, func() { GetOrCreateHistogram(name) }) +} + +func TestWritePrometheusSerial(t *testing.T) { + if err := testWritePrometheus(); err != nil { + t.Fatal(err) + } +} + +func TestWritePrometheusConcurrent(t *testing.T) { + if err := testConcurrent(testWritePrometheus); err != nil { + t.Fatal(err) + } +} + +func testWritePrometheus() error { + var bb bytes.Buffer + WritePrometheus(&bb, false) + resultWithoutProcessMetrics := bb.String() + bb.Reset() + WritePrometheus(&bb, true) + resultWithProcessMetrics := bb.String() + if len(resultWithProcessMetrics) <= len(resultWithoutProcessMetrics) { + return fmt.Errorf("result with process metrics must contain more data than the result without process metrics; got\n%q\nvs\n%q", + resultWithProcessMetrics, resultWithoutProcessMetrics) + } + return nil +} + +func expectPanic(t *testing.T, context string, f func()) { + t.Helper() + defer func() { + t.Helper() + if r := recover(); r == nil { + t.Fatalf("expecting panic in %s", context) + } + }() + f() +} + +func testConcurrent(f func() error) error { + const concurrency = 5 + resultsCh := make(chan error, concurrency) + for i := 0; i < concurrency; i++ { + go func() { + resultsCh <- f() + }() + } + for i := 0; i < concurrency; i++ { + select { + case err := <-resultsCh: + if err != nil { + return fmt.Errorf("unexpected error: %s", err) + } + case <-time.After(time.Second * 5): + return fmt.Errorf("timeout") + } + } + return nil +} + +func testMarshalTo(t *testing.T, m metric, prefix, resultExpected string) { + t.Helper() + var bb bytes.Buffer + m.marshalTo(prefix, &bb) + result := bb.String() + if result != resultExpected { + t.Fatalf("unexpected marshaled metric;\ngot\n%q\nwant\n%q", result, resultExpected) + } +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go new file mode 100644 index 0000000..12b5de8 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go @@ -0,0 +1,265 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strconv" + "strings" + "time" +) + +// See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 . +const userHZ = 100 + +// See http://man7.org/linux/man-pages/man5/proc.5.html +type procStat struct { + State byte + Ppid int + Pgrp int + Session int + TtyNr int + Tpgid int + Flags uint + Minflt uint + Cminflt uint + Majflt uint + Cmajflt uint + Utime uint + Stime uint + Cutime int + Cstime int + Priority int + Nice int + NumThreads int + ItrealValue int + Starttime uint64 + Vsize uint + Rss int +} + +func writeProcessMetrics(w io.Writer) { + statFilepath := "/proc/self/stat" + data, err := ioutil.ReadFile(statFilepath) + if err != nil { + log.Printf("ERROR: cannot open %s: %s", statFilepath, err) + return + } + // Search for the end of command. + n := bytes.LastIndex(data, []byte(") ")) + if n < 0 { + log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath) + return + } + data = data[n+2:] + + var p procStat + bb := bytes.NewBuffer(data) + _, err = fmt.Fscanf(bb, "%c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", + &p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt, + &p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss) + if err != nil { + log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err) + return + } + + // It is expensive obtaining `process_open_fds` when big number of file descriptors is opened, + // so don't do it here. + // See writeFDMetrics instead. + + utime := float64(p.Utime) / userHZ + stime := float64(p.Stime) / userHZ + fmt.Fprintf(w, "process_cpu_seconds_system_total %g\n", stime) + fmt.Fprintf(w, "process_cpu_seconds_total %g\n", utime+stime) + fmt.Fprintf(w, "process_cpu_seconds_user_total %g\n", utime) + fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt) + fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt) + fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads) + fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096) + fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds) + fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize) + writeProcessMemMetrics(w) + writeIOMetrics(w) +} + +func writeIOMetrics(w io.Writer) { + ioFilepath := "/proc/self/io" + data, err := ioutil.ReadFile(ioFilepath) + if err != nil { + log.Printf("ERROR: cannot open %q: %s", ioFilepath, err) + } + getInt := func(s string) int64 { + n := strings.IndexByte(s, ' ') + if n < 0 { + log.Printf("ERROR: cannot find whitespace in %q at %q", s, ioFilepath) + return 0 + } + v, err := strconv.ParseInt(s[n+1:], 10, 64) + if err != nil { + log.Printf("ERROR: cannot parse %q at %q: %s", s, ioFilepath, err) + return 0 + } + return v + } + var rchar, wchar, syscr, syscw, readBytes, writeBytes int64 + lines := strings.Split(string(data), "\n") + for _, s := range lines { + s = strings.TrimSpace(s) + switch { + case strings.HasPrefix(s, "rchar: "): + rchar = getInt(s) + case strings.HasPrefix(s, "wchar: "): + wchar = getInt(s) + case strings.HasPrefix(s, "syscr: "): + syscr = getInt(s) + case strings.HasPrefix(s, "syscw: "): + syscw = getInt(s) + case strings.HasPrefix(s, "read_bytes: "): + readBytes = getInt(s) + case strings.HasPrefix(s, "write_bytes: "): + writeBytes = getInt(s) + } + } + fmt.Fprintf(w, "process_io_read_bytes_total %d\n", rchar) + fmt.Fprintf(w, "process_io_written_bytes_total %d\n", wchar) + fmt.Fprintf(w, "process_io_read_syscalls_total %d\n", syscr) + fmt.Fprintf(w, "process_io_write_syscalls_total %d\n", syscw) + fmt.Fprintf(w, "process_io_storage_read_bytes_total %d\n", readBytes) + fmt.Fprintf(w, "process_io_storage_written_bytes_total %d\n", writeBytes) +} + +var startTimeSeconds = time.Now().Unix() + +// writeFDMetrics writes process_max_fds and process_open_fds metrics to w. +func writeFDMetrics(w io.Writer) { + totalOpenFDs, err := getOpenFDsCount("/proc/self/fd") + if err != nil { + log.Printf("ERROR: cannot determine open file descriptors count: %s", err) + return + } + maxOpenFDs, err := getMaxFilesLimit("/proc/self/limits") + if err != nil { + log.Printf("ERROR: cannot determine the limit on open file descritors: %s", err) + return + } + fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs) + fmt.Fprintf(w, "process_open_fds %d\n", totalOpenFDs) +} + +func getOpenFDsCount(path string) (uint64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + var totalOpenFDs uint64 + for { + names, err := f.Readdirnames(512) + if err == io.EOF { + break + } + if err != nil { + return 0, fmt.Errorf("unexpected error at Readdirnames: %s", err) + } + totalOpenFDs += uint64(len(names)) + } + return totalOpenFDs, nil +} + +func getMaxFilesLimit(path string) (uint64, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return 0, err + } + lines := strings.Split(string(data), "\n") + const prefix = "Max open files" + for _, s := range lines { + if !strings.HasPrefix(s, prefix) { + continue + } + text := strings.TrimSpace(s[len(prefix):]) + // Extract soft limit. + n := strings.IndexByte(text, ' ') + if n < 0 { + return 0, fmt.Errorf("cannot extract soft limit from %q", s) + } + text = text[:n] + if text == "unlimited" { + return 1<<64 - 1, nil + } + limit, err := strconv.ParseUint(text, 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse soft limit from %q: %s", s, err) + } + return limit, nil + } + return 0, fmt.Errorf("cannot find max open files limit") +} + +// https://man7.org/linux/man-pages/man5/procfs.5.html +type memStats struct { + vmPeak uint64 + rssPeak uint64 + rssAnon uint64 + rssFile uint64 + rssShmem uint64 +} + +func writeProcessMemMetrics(w io.Writer) { + ms, err := getMemStats("/proc/self/status") + if err != nil { + log.Printf("ERROR: cannot determine memory status: %s", err) + return + } + fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak) + fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", ms.rssPeak) + fmt.Fprintf(w, "process_resident_memory_anon_bytes %d\n", ms.rssAnon) + fmt.Fprintf(w, "process_resident_memory_file_bytes %d\n", ms.rssFile) + fmt.Fprintf(w, "process_resident_memory_shared_bytes %d\n", ms.rssShmem) + +} + +func getMemStats(path string) (*memStats, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var ms memStats + lines := strings.Split(string(data), "\n") + for _, s := range lines { + if !strings.HasPrefix(s, "Vm") && !strings.HasPrefix(s, "Rss") { + continue + } + // Extract key value. + line := strings.Fields(s) + if len(line) != 3 { + return nil, fmt.Errorf("unexpected number of fields found in %q; got %d; want %d", s, len(line), 3) + } + memStatName := line[0] + memStatValue := line[1] + value, err := strconv.ParseUint(memStatValue, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse number from %q: %w", s, err) + } + if line[2] != "kB" { + return nil, fmt.Errorf("expecting kB value in %q; got %q", s, line[2]) + } + value *= 1024 + switch memStatName { + case "VmPeak:": + ms.vmPeak = value + case "VmHWM:": + ms.rssPeak = value + case "RssAnon:": + ms.rssAnon = value + case "RssFile:": + ms.rssFile = value + case "RssShmem:": + ms.rssShmem = value + } + } + return &ms, nil +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux_test.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux_test.go new file mode 100644 index 0000000..88dac03 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux_test.go @@ -0,0 +1,51 @@ +package metrics + +import "testing" + +func TestGetMaxFilesLimit(t *testing.T) { + f := func(want uint64, path string, wantErr bool) { + t.Helper() + got, err := getMaxFilesLimit(path) + if err != nil && !wantErr { + t.Fatalf("unexpected error: %v", err) + } + if got != want { + t.Fatalf("unexpected result: %d, want: %d at getMaxFilesLimit", got, want) + } + + } + f(1024, "testdata/limits", false) + f(0, "testdata/bad_path", true) + f(0, "testdata/limits_bad", true) +} + +func TestGetOpenFDsCount(t *testing.T) { + f := func(want uint64, path string, wantErr bool) { + t.Helper() + got, err := getOpenFDsCount(path) + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unexpected error: %v", err) + } + if got != want { + t.Fatalf("unexpected result: %d, want: %d at getOpenFDsCount", got, want) + } + } + f(5, "testdata/fd/", false) + f(0, "testdata/fd/0", true) + f(0, "testdata/limits", true) +} + +func TestGetMemStats(t *testing.T) { + f := func(want memStats, path string, wantErr bool) { + t.Helper() + got, err := getMemStats(path) + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unexpected error: %v", err) + } + if got != nil && *got != want { + t.Fatalf("unexpected result: %d, want: %d at getMemStats", *got, want) + } + } + f(memStats{vmPeak: 2130489344, rssPeak: 200679424, rssAnon: 121602048, rssFile: 11362304}, "testdata/status", false) + f(memStats{}, "testdata/status_bad", true) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go new file mode 100644 index 0000000..5e6ac93 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go @@ -0,0 +1,15 @@ +// +build !linux + +package metrics + +import ( + "io" +) + +func writeProcessMetrics(w io.Writer) { + // TODO: implement it +} + +func writeFDMetrics(w io.Writer) { + // TODO: implement it. +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/set.go b/vendor/github.com/VictoriaMetrics/metrics/set.go new file mode 100644 index 0000000..69b4de8 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/set.go @@ -0,0 +1,519 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "sort" + "sync" + "time" +) + +// Set is a set of metrics. +// +// Metrics belonging to a set are exported separately from global metrics. +// +// Set.WritePrometheus must be called for exporting metrics from the set. +type Set struct { + mu sync.Mutex + a []*namedMetric + m map[string]*namedMetric + summaries []*Summary +} + +// NewSet creates new set of metrics. +func NewSet() *Set { + return &Set{ + m: make(map[string]*namedMetric), + } +} + +// WritePrometheus writes all the metrics from s to w in Prometheus format. +func (s *Set) WritePrometheus(w io.Writer) { + // Collect all the metrics in in-memory buffer in order to prevent from long locking due to slow w. + var bb bytes.Buffer + lessFunc := func(i, j int) bool { + return s.a[i].name < s.a[j].name + } + s.mu.Lock() + for _, sm := range s.summaries { + sm.updateQuantiles() + } + if !sort.SliceIsSorted(s.a, lessFunc) { + sort.Slice(s.a, lessFunc) + } + sa := append([]*namedMetric(nil), s.a...) + s.mu.Unlock() + + // Call marshalTo without the global lock, since certain metric types such as Gauge + // can call a callback, which, in turn, can try calling s.mu.Lock again. + for _, nm := range sa { + nm.metric.marshalTo(nm.name, &bb) + } + w.Write(bb.Bytes()) +} + +// NewHistogram creates and returns new histogram in s with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned histogram is safe to use from concurrent goroutines. +func (s *Set) NewHistogram(name string) *Histogram { + h := &Histogram{} + s.registerMetric(name, h) + return h +} + +// GetOrCreateHistogram returns registered histogram in s with the given name +// or creates new histogram if s doesn't contain histogram with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned histogram is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewHistogram instead of GetOrCreateHistogram. +func (s *Set) GetOrCreateHistogram(name string) *Histogram { + s.mu.Lock() + nm := s.m[name] + s.mu.Unlock() + if nm == nil { + // Slow path - create and register missing histogram. + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + nmNew := &namedMetric{ + name: name, + metric: &Histogram{}, + } + s.mu.Lock() + nm = s.m[name] + if nm == nil { + nm = nmNew + s.m[name] = nm + s.a = append(s.a, nm) + } + s.mu.Unlock() + } + h, ok := nm.metric.(*Histogram) + if !ok { + panic(fmt.Errorf("BUG: metric %q isn't a Histogram. It is %T", name, nm.metric)) + } + return h +} + +// NewCounter registers and returns new counter with the given name in the s. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +func (s *Set) NewCounter(name string) *Counter { + c := &Counter{} + s.registerMetric(name, c) + return c +} + +// GetOrCreateCounter returns registered counter in s with the given name +// or creates new counter if s doesn't contain counter with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewCounter instead of GetOrCreateCounter. +func (s *Set) GetOrCreateCounter(name string) *Counter { + s.mu.Lock() + nm := s.m[name] + s.mu.Unlock() + if nm == nil { + // Slow path - create and register missing counter. + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + nmNew := &namedMetric{ + name: name, + metric: &Counter{}, + } + s.mu.Lock() + nm = s.m[name] + if nm == nil { + nm = nmNew + s.m[name] = nm + s.a = append(s.a, nm) + } + s.mu.Unlock() + } + c, ok := nm.metric.(*Counter) + if !ok { + panic(fmt.Errorf("BUG: metric %q isn't a Counter. It is %T", name, nm.metric)) + } + return c +} + +// NewFloatCounter registers and returns new FloatCounter with the given name in the s. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned FloatCounter is safe to use from concurrent goroutines. +func (s *Set) NewFloatCounter(name string) *FloatCounter { + c := &FloatCounter{} + s.registerMetric(name, c) + return c +} + +// GetOrCreateFloatCounter returns registered FloatCounter in s with the given name +// or creates new FloatCounter if s doesn't contain FloatCounter with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned FloatCounter is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewFloatCounter instead of GetOrCreateFloatCounter. +func (s *Set) GetOrCreateFloatCounter(name string) *FloatCounter { + s.mu.Lock() + nm := s.m[name] + s.mu.Unlock() + if nm == nil { + // Slow path - create and register missing counter. + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + nmNew := &namedMetric{ + name: name, + metric: &FloatCounter{}, + } + s.mu.Lock() + nm = s.m[name] + if nm == nil { + nm = nmNew + s.m[name] = nm + s.a = append(s.a, nm) + } + s.mu.Unlock() + } + c, ok := nm.metric.(*FloatCounter) + if !ok { + panic(fmt.Errorf("BUG: metric %q isn't a Counter. It is %T", name, nm.metric)) + } + return c +} + +// NewGauge registers and returns gauge with the given name in s, which calls f +// to obtain gauge value. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// f must be safe for concurrent calls. +// +// The returned gauge is safe to use from concurrent goroutines. +func (s *Set) NewGauge(name string, f func() float64) *Gauge { + if f == nil { + panic(fmt.Errorf("BUG: f cannot be nil")) + } + g := &Gauge{ + f: f, + } + s.registerMetric(name, g) + return g +} + +// GetOrCreateGauge returns registered gauge with the given name in s +// or creates new gauge if s doesn't contain gauge with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned gauge is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewGauge instead of GetOrCreateGauge. +func (s *Set) GetOrCreateGauge(name string, f func() float64) *Gauge { + s.mu.Lock() + nm := s.m[name] + s.mu.Unlock() + if nm == nil { + // Slow path - create and register missing gauge. + if f == nil { + panic(fmt.Errorf("BUG: f cannot be nil")) + } + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + nmNew := &namedMetric{ + name: name, + metric: &Gauge{ + f: f, + }, + } + s.mu.Lock() + nm = s.m[name] + if nm == nil { + nm = nmNew + s.m[name] = nm + s.a = append(s.a, nm) + } + s.mu.Unlock() + } + g, ok := nm.metric.(*Gauge) + if !ok { + panic(fmt.Errorf("BUG: metric %q isn't a Gauge. It is %T", name, nm.metric)) + } + return g +} + +// NewSummary creates and returns new summary with the given name in s. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func (s *Set) NewSummary(name string) *Summary { + return s.NewSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) +} + +// NewSummaryExt creates and returns new summary in s with the given name, +// window and quantiles. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func (s *Set) NewSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + sm := newSummary(window, quantiles) + + s.mu.Lock() + // defer will unlock in case of panic + // checks in tests + defer s.mu.Unlock() + + s.mustRegisterLocked(name, sm) + registerSummaryLocked(sm) + s.registerSummaryQuantilesLocked(name, sm) + s.summaries = append(s.summaries, sm) + return sm +} + +// GetOrCreateSummary returns registered summary with the given name in s +// or creates new summary if s doesn't contain summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewSummary instead of GetOrCreateSummary. +func (s *Set) GetOrCreateSummary(name string) *Summary { + return s.GetOrCreateSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) +} + +// GetOrCreateSummaryExt returns registered summary with the given name, +// window and quantiles in s or creates new summary if s doesn't +// contain summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewSummaryExt instead of GetOrCreateSummaryExt. +func (s *Set) GetOrCreateSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + s.mu.Lock() + nm := s.m[name] + s.mu.Unlock() + if nm == nil { + // Slow path - create and register missing summary. + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + sm := newSummary(window, quantiles) + nmNew := &namedMetric{ + name: name, + metric: sm, + } + s.mu.Lock() + nm = s.m[name] + if nm == nil { + nm = nmNew + s.m[name] = nm + s.a = append(s.a, nm) + registerSummaryLocked(sm) + s.registerSummaryQuantilesLocked(name, sm) + } + s.summaries = append(s.summaries, sm) + s.mu.Unlock() + } + sm, ok := nm.metric.(*Summary) + if !ok { + panic(fmt.Errorf("BUG: metric %q isn't a Summary. It is %T", name, nm.metric)) + } + if sm.window != window { + panic(fmt.Errorf("BUG: invalid window requested for the summary %q; requested %s; need %s", name, window, sm.window)) + } + if !isEqualQuantiles(sm.quantiles, quantiles) { + panic(fmt.Errorf("BUG: invalid quantiles requested from the summary %q; requested %v; need %v", name, quantiles, sm.quantiles)) + } + return sm +} + +func (s *Set) registerSummaryQuantilesLocked(name string, sm *Summary) { + for i, q := range sm.quantiles { + quantileValueName := addTag(name, fmt.Sprintf(`quantile="%g"`, q)) + qv := &quantileValue{ + sm: sm, + idx: i, + } + s.mustRegisterLocked(quantileValueName, qv) + } +} + +func (s *Set) registerMetric(name string, m metric) { + if err := validateMetric(name); err != nil { + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + s.mu.Lock() + // defer will unlock in case of panic + // checks in test + defer s.mu.Unlock() + s.mustRegisterLocked(name, m) +} + +// mustRegisterLocked registers given metric with +// the given name. Panics if the given name was +// already registered before. +func (s *Set) mustRegisterLocked(name string, m metric) { + nm, ok := s.m[name] + if !ok { + nm = &namedMetric{ + name: name, + metric: m, + } + s.m[name] = nm + s.a = append(s.a, nm) + } + if ok { + panic(fmt.Errorf("BUG: metric %q is already registered", name)) + } +} + +// UnregisterMetric removes metric with the given name from s. +// +// True is returned if the metric has been removed. +// False is returned if the given metric is missing in s. +func (s *Set) UnregisterMetric(name string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + nm, ok := s.m[name] + if !ok { + return false + } + m := nm.metric + + delete(s.m, name) + + deleteFromList := func(metricName string) { + for i, nm := range s.a { + if nm.name == metricName { + s.a = append(s.a[:i], s.a[i+1:]...) + return + } + } + panic(fmt.Errorf("BUG: cannot find metric %q in the list of registered metrics", name)) + } + + // remove metric from s.a + deleteFromList(name) + + sm, ok := m.(*Summary) + if !ok { + // There is no need in cleaning up summary. + return true + } + + // cleanup registry from per-quantile metrics + for _, q := range sm.quantiles { + quantileValueName := addTag(name, fmt.Sprintf(`quantile="%g"`, q)) + delete(s.m, quantileValueName) + deleteFromList(quantileValueName) + } + + // Remove sm from s.summaries + found := false + for i, xsm := range s.summaries { + if xsm == sm { + s.summaries = append(s.summaries[:i], s.summaries[i+1:]...) + found = true + break + } + } + if !found { + panic(fmt.Errorf("BUG: cannot find summary %q in the list of registered summaries", name)) + } + unregisterSummary(sm) + return true +} + +// ListMetricNames returns a list of all the metrics in s. +func (s *Set) ListMetricNames() []string { + var list []string + for name := range s.m { + list = append(list, name) + } + return list +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/set_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/set_example_test.go new file mode 100644 index 0000000..50845d3 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/set_example_test.go @@ -0,0 +1,25 @@ +package metrics_test + +import ( + "bytes" + "fmt" + "github.com/VictoriaMetrics/metrics" +) + +func ExampleSet() { + // Create a set with a counter + s := metrics.NewSet() + sc := s.NewCounter("set_counter") + sc.Inc() + s.NewGauge(`set_gauge{foo="bar"}`, func() float64 { return 42 }) + + // Dump metrics from s. + var bb bytes.Buffer + s.WritePrometheus(&bb) + fmt.Printf("set metrics:\n%s\n", bb.String()) + + // Output: + // set metrics: + // set_counter 1 + // set_gauge{foo="bar"} 42 +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/set_test.go b/vendor/github.com/VictoriaMetrics/metrics/set_test.go new file mode 100644 index 0000000..c954d87 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/set_test.go @@ -0,0 +1,152 @@ +package metrics + +import ( + "fmt" + "sync" + "testing" + "time" +) + +func TestNewSet(t *testing.T) { + var ss []*Set + for i := 0; i < 10; i++ { + s := NewSet() + ss = append(ss, s) + } + for i := 0; i < 10; i++ { + s := ss[i] + for j := 0; j < 10; j++ { + c := s.NewCounter(fmt.Sprintf("counter_%d", j)) + c.Inc() + if n := c.Get(); n != 1 { + t.Fatalf("unexpected counter value; got %d; want %d", n, 1) + } + g := s.NewGauge(fmt.Sprintf("gauge_%d", j), func() float64 { return 123 }) + if v := g.Get(); v != 123 { + t.Fatalf("unexpected gauge value; got %v; want %v", v, 123) + } + sm := s.NewSummary(fmt.Sprintf("summary_%d", j)) + if sm == nil { + t.Fatalf("NewSummary returned nil") + } + h := s.NewHistogram(fmt.Sprintf("histogram_%d", j)) + if h == nil { + t.Fatalf("NewHistogram returned nil") + } + } + } +} + +func TestSetListMetricNames(t *testing.T) { + s := NewSet() + expect := []string{"cnt1", "cnt2", "cnt3"} + // Initialize a few counters + for _, n := range expect { + c := s.NewCounter(n) + c.Inc() + } + + list := s.ListMetricNames() + + if len(list) != len(expect) { + t.Fatalf("Metrics count is wrong for listing") + } + for _, e := range expect { + found := false + for _, n := range list { + if e == n { + found = true + } + } + if !found { + t.Fatalf("Metric %s not found in listing", e) + } + } +} + +func TestSetUnregisterMetric(t *testing.T) { + s := NewSet() + const cName, smName = "counter_1", "summary_1" + // Initialize a few metrics + c := s.NewCounter(cName) + c.Inc() + sm := s.NewSummary(smName) + sm.Update(1) + + // Unregister existing metrics + if !s.UnregisterMetric(cName) { + t.Fatalf("UnregisterMetric(%s) must return true", cName) + } + if !s.UnregisterMetric(smName) { + t.Fatalf("UnregisterMetric(%s) must return true", smName) + } + + // Unregister twice must return false + if s.UnregisterMetric(cName) { + t.Fatalf("UnregisterMetric(%s) must return false on unregistered metric", cName) + } + if s.UnregisterMetric(smName) { + t.Fatalf("UnregisterMetric(%s) must return false on unregistered metric", smName) + } + + // verify that registry is empty + if len(s.m) != 0 { + t.Fatalf("expected metrics map to be empty; got %d elements", len(s.m)) + } + if len(s.a) != 0 { + t.Fatalf("expected metrics list to be empty; got %d elements", len(s.a)) + } + + // Validate metrics are removed + ok := false + for _, n := range s.ListMetricNames() { + if n == cName || n == smName { + ok = true + } + } + if ok { + t.Fatalf("Metric counter_1 and summary_1 must not be listed anymore after unregister") + } + + // re-register with the same names supposed + // to be successful + s.NewCounter(cName).Inc() + s.NewSummary(smName).Update(float64(1)) +} + +// TestRegisterUnregister tests concurrent access to +// metrics during registering and unregistering. +// Should be tested specifically with `-race` enabled. +func TestRegisterUnregister(t *testing.T) { + const ( + workers = 16 + iterations = 1e3 + ) + wg := sync.WaitGroup{} + wg.Add(workers) + for n := 0; n < workers; n++ { + go func() { + defer wg.Done() + now := time.Now() + for i := 0; i < iterations; i++ { + iteration := i % 5 + counter := fmt.Sprintf(`counter{iteration="%d"}`, iteration) + GetOrCreateCounter(counter).Add(i) + UnregisterMetric(counter) + + histogram := fmt.Sprintf(`histogram{iteration="%d"}`, iteration) + GetOrCreateHistogram(histogram).UpdateDuration(now) + UnregisterMetric(histogram) + + gauge := fmt.Sprintf(`gauge{iteration="%d"}`, iteration) + GetOrCreateGauge(gauge, func() float64 { return 1 }) + UnregisterMetric(gauge) + + summary := fmt.Sprintf(`summary{iteration="%d"}`, iteration) + GetOrCreateSummary(summary).Update(float64(i)) + UnregisterMetric(summary) + } + }() + } + wg.Wait() +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/summary.go b/vendor/github.com/VictoriaMetrics/metrics/summary.go new file mode 100644 index 0000000..0f01e9a --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/summary.go @@ -0,0 +1,254 @@ +package metrics + +import ( + "fmt" + "io" + "math" + "strings" + "sync" + "time" + + "github.com/valyala/histogram" +) + +const defaultSummaryWindow = 5 * time.Minute + +var defaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} + +// Summary implements summary. +type Summary struct { + mu sync.Mutex + + curr *histogram.Fast + next *histogram.Fast + + quantiles []float64 + quantileValues []float64 + + sum float64 + count uint64 + + window time.Duration +} + +// NewSummary creates and returns new summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummary(name string) *Summary { + return defaultSet.NewSummary(name) +} + +// NewSummaryExt creates and returns new summary with the given name, +// window and quantiles. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + return defaultSet.NewSummaryExt(name, window, quantiles) +} + +func newSummary(window time.Duration, quantiles []float64) *Summary { + // Make a copy of quantiles in order to prevent from their modification by the caller. + quantiles = append([]float64{}, quantiles...) + validateQuantiles(quantiles) + sm := &Summary{ + curr: histogram.NewFast(), + next: histogram.NewFast(), + quantiles: quantiles, + quantileValues: make([]float64, len(quantiles)), + window: window, + } + return sm +} + +func validateQuantiles(quantiles []float64) { + for _, q := range quantiles { + if q < 0 || q > 1 { + panic(fmt.Errorf("BUG: quantile must be in the range [0..1]; got %v", q)) + } + } +} + +// Update updates the summary. +func (sm *Summary) Update(v float64) { + sm.mu.Lock() + sm.curr.Update(v) + sm.next.Update(v) + sm.sum += v + sm.count++ + sm.mu.Unlock() +} + +// UpdateDuration updates request duration based on the given startTime. +func (sm *Summary) UpdateDuration(startTime time.Time) { + d := time.Since(startTime).Seconds() + sm.Update(d) +} + +func (sm *Summary) marshalTo(prefix string, w io.Writer) { + // Marshal only *_sum and *_count values. + // Quantile values should be already updated by the caller via sm.updateQuantiles() call. + // sm.quantileValues will be marshaled later via quantileValue.marshalTo. + sm.mu.Lock() + sum := sm.sum + count := sm.count + sm.mu.Unlock() + + if count > 0 { + name, filters := splitMetricName(prefix) + if float64(int64(sum)) == sum { + // Marshal integer sum without scientific notation + fmt.Fprintf(w, "%s_sum%s %d\n", name, filters, int64(sum)) + } else { + fmt.Fprintf(w, "%s_sum%s %g\n", name, filters, sum) + } + fmt.Fprintf(w, "%s_count%s %d\n", name, filters, count) + } +} + +func splitMetricName(name string) (string, string) { + n := strings.IndexByte(name, '{') + if n < 0 { + return name, "" + } + return name[:n], name[n:] +} + +func (sm *Summary) updateQuantiles() { + sm.mu.Lock() + sm.quantileValues = sm.curr.Quantiles(sm.quantileValues[:0], sm.quantiles) + sm.mu.Unlock() +} + +// GetOrCreateSummary returns registered summary with the given name +// or creates new summary if the registry doesn't contain summary with +// the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewSummary instead of GetOrCreateSummary. +func GetOrCreateSummary(name string) *Summary { + return defaultSet.GetOrCreateSummary(name) +} + +// GetOrCreateSummaryExt returns registered summary with the given name, +// window and quantiles or creates new summary if the registry doesn't +// contain summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +// +// Performance tip: prefer NewSummaryExt instead of GetOrCreateSummaryExt. +func GetOrCreateSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + return defaultSet.GetOrCreateSummaryExt(name, window, quantiles) +} + +func isEqualQuantiles(a, b []float64) bool { + // Do not use relfect.DeepEqual, since it is slower than the direct comparison. + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +type quantileValue struct { + sm *Summary + idx int +} + +func (qv *quantileValue) marshalTo(prefix string, w io.Writer) { + qv.sm.mu.Lock() + v := qv.sm.quantileValues[qv.idx] + qv.sm.mu.Unlock() + if !math.IsNaN(v) { + fmt.Fprintf(w, "%s %g\n", prefix, v) + } +} + +func addTag(name, tag string) string { + if len(name) == 0 || name[len(name)-1] != '}' { + return fmt.Sprintf("%s{%s}", name, tag) + } + return fmt.Sprintf("%s,%s}", name[:len(name)-1], tag) +} + +func registerSummaryLocked(sm *Summary) { + window := sm.window + summariesLock.Lock() + summaries[window] = append(summaries[window], sm) + if len(summaries[window]) == 1 { + go summariesSwapCron(window) + } + summariesLock.Unlock() +} + +func unregisterSummary(sm *Summary) { + window := sm.window + summariesLock.Lock() + sms := summaries[window] + found := false + for i, xsm := range sms { + if xsm == sm { + sms = append(sms[:i], sms[i+1:]...) + found = true + break + } + } + if !found { + panic(fmt.Errorf("BUG: cannot find registered summary %p", sm)) + } + summaries[window] = sms + summariesLock.Unlock() +} + +func summariesSwapCron(window time.Duration) { + for { + time.Sleep(window / 2) + summariesLock.Lock() + for _, sm := range summaries[window] { + sm.mu.Lock() + tmp := sm.curr + sm.curr = sm.next + sm.next = tmp + sm.next.Reset() + sm.mu.Unlock() + } + summariesLock.Unlock() + } +} + +var ( + summaries = map[time.Duration][]*Summary{} + summariesLock sync.Mutex +) diff --git a/vendor/github.com/VictoriaMetrics/metrics/summary_example_test.go b/vendor/github.com/VictoriaMetrics/metrics/summary_example_test.go new file mode 100644 index 0000000..09dc57f --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/summary_example_test.go @@ -0,0 +1,31 @@ +package metrics_test + +import ( + "fmt" + "time" + + "github.com/VictoriaMetrics/metrics" +) + +func ExampleSummary() { + // Define a summary in global scope. + var s = metrics.NewSummary(`request_duration_seconds{path="/foo/bar"}`) + + // Update the summary with the duration of processRequest call. + startTime := time.Now() + processRequest() + s.UpdateDuration(startTime) +} + +func ExampleSummary_vec() { + for i := 0; i < 3; i++ { + // Dynamically construct metric name and pass it to GetOrCreateSummary. + name := fmt.Sprintf(`response_size_bytes{path=%q}`, "/foo/bar") + response := processRequest() + metrics.GetOrCreateSummary(name).Update(float64(len(response))) + } +} + +func processRequest() string { + return "foobar" +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/summary_test.go b/vendor/github.com/VictoriaMetrics/metrics/summary_test.go new file mode 100644 index 0000000..9182a65 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/summary_test.go @@ -0,0 +1,155 @@ +package metrics + +import ( + "bytes" + "fmt" + "strings" + "testing" + "time" +) + +func TestSummarySerial(t *testing.T) { + name := `TestSummarySerial` + s := NewSummary(name) + + // Verify that the summary isn't visible in the output of WritePrometheus, + // since it doesn't contain any values yet. + var bb bytes.Buffer + WritePrometheus(&bb, false) + result := bb.String() + if strings.Contains(result, name) { + t.Fatalf("summary %s shouldn't be visible in the WritePrometheus output; got\n%s", name, result) + } + + // Write data to summary + for i := 0; i < 2000; i++ { + s.Update(float64(i)) + t := time.Now() + s.UpdateDuration(t.Add(-time.Millisecond * time.Duration(i))) + } + + // Make sure the summary prints _sum and _count on marshalTo call + testMarshalTo(t, s, "prefix", fmt.Sprintf("prefix_sum %g\nprefix_count %d\n", s.sum, s.count)) + testMarshalTo(t, s, `m{foo="bar"}`, fmt.Sprintf("m_sum{foo=\"bar\"} %g\nm_count{foo=\"bar\"} %d\n", s.sum, s.count)) + + // Verify s.quantileValues + s.updateQuantiles() + if s.quantileValues[len(s.quantileValues)-1] != 1999 { + t.Fatalf("unexpected quantileValues[last]; got %v; want %v", s.quantileValues[len(s.quantileValues)-1], 1999) + } + + // Make sure the summary becomes visible in the output of WritePrometheus, + // since now it contains values. + bb.Reset() + WritePrometheus(&bb, false) + result = bb.String() + if !strings.Contains(result, name) { + t.Fatalf("missing summary %s in the WritePrometheus output; got\n%s", name, result) + } +} + +func TestSummaryConcurrent(t *testing.T) { + name := "SummaryConcurrent" + s := NewSummary(name) + err := testConcurrent(func() error { + for i := 0; i < 10; i++ { + s.Update(float64(i)) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + testMarshalTo(t, s, "prefix", "prefix_sum 225\nprefix_count 50\n") +} + +func TestSummaryWithTags(t *testing.T) { + name := `TestSummary{tag="foo"}` + s := NewSummary(name) + s.Update(123) + + var bb bytes.Buffer + WritePrometheus(&bb, false) + result := bb.String() + namePrefixWithTag := `TestSummary{tag="foo",quantile="` + if !strings.Contains(result, namePrefixWithTag) { + t.Fatalf("missing summary prefix %s in the WritePrometheus output; got\n%s", namePrefixWithTag, result) + } +} + +func TestSummaryInvalidQuantiles(t *testing.T) { + name := "SummaryInvalidQuantiles" + expectPanic(t, name, func() { + NewSummaryExt(name, time.Minute, []float64{123, -234}) + }) +} + +func TestSummarySmallWindow(t *testing.T) { + name := "SummarySmallWindow" + window := time.Millisecond * 20 + quantiles := []float64{0.1, 0.2, 0.3} + s := NewSummaryExt(name, window, quantiles) + for i := 0; i < 2000; i++ { + s.Update(123) + } + // Wait for window update and verify that the summary has been cleared. + time.Sleep(2 * window) + var bb bytes.Buffer + WritePrometheus(&bb, false) + result := bb.String() + // _sum and _count are present in the output. + // Only {quantile} shouldn't be present. + name += "{" + if strings.Contains(result, name) { + t.Fatalf("summary %s cannot be present in the WritePrometheus output; got\n%s", name, result) + } +} + +func TestGetOrCreateSummaryInvalidWindow(t *testing.T) { + name := "GetOrCreateSummaryInvalidWindow" + GetOrCreateSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) + expectPanic(t, name, func() { + GetOrCreateSummaryExt(name, defaultSummaryWindow/2, defaultSummaryQuantiles) + }) +} + +func TestGetOrCreateSummaryInvalidQuantiles(t *testing.T) { + name := "GetOrCreateSummaryInvalidQuantiles" + GetOrCreateSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) + expectPanic(t, name, func() { + GetOrCreateSummaryExt(name, defaultSummaryWindow, []float64{0.1, 0.2}) + }) + quantiles := append([]float64{}, defaultSummaryQuantiles...) + quantiles[len(quantiles)-1] /= 2 + expectPanic(t, name, func() { + GetOrCreateSummaryExt(name, defaultSummaryWindow, quantiles) + }) +} + +func TestGetOrCreateSummarySerial(t *testing.T) { + name := "GetOrCreateSummarySerial" + if err := testGetOrCreateSummary(name); err != nil { + t.Fatal(err) + } +} + +func TestGetOrCreateSummaryConcurrent(t *testing.T) { + name := "GetOrCreateSummaryConcurrent" + err := testConcurrent(func() error { + return testGetOrCreateSummary(name) + }) + if err != nil { + t.Fatal(err) + } +} + +func testGetOrCreateSummary(name string) error { + s1 := GetOrCreateSummary(name) + for i := 0; i < 10; i++ { + s2 := GetOrCreateSummary(name) + if s1 != s2 { + return fmt.Errorf("unexpected summary returned; got %p; want %p", s2, s1) + } + } + return nil +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/0 b/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/0 new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/10 b/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/10 new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/2 b/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/2 new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/3 b/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/3 new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/5 b/vendor/github.com/VictoriaMetrics/metrics/testdata/fd/5 new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/limits b/vendor/github.com/VictoriaMetrics/metrics/testdata/limits new file mode 100644 index 0000000..fb520d3 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/testdata/limits @@ -0,0 +1,17 @@ +Limit Soft Limit Hard Limit Units +Max cpu time unlimited unlimited seconds +Max file size unlimited unlimited bytes +Max data size unlimited unlimited bytes +Max stack size 8388608 unlimited bytes +Max core file size 0 unlimited bytes +Max resident set unlimited unlimited bytes +Max processes 127458 127458 processes +Max open files 1024 1048576 files +Max locked memory 67108864 67108864 bytes +Max address space unlimited unlimited bytes +Max file locks unlimited unlimited locks +Max pending signals 127458 127458 signals +Max msgqueue size 819200 819200 bytes +Max nice priority 0 0 +Max realtime priority 0 0 +Max realtime timeout unlimited unlimited us \ No newline at end of file diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/limits_bad b/vendor/github.com/VictoriaMetrics/metrics/testdata/limits_bad new file mode 100644 index 0000000..d54162d --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/testdata/limits_bad @@ -0,0 +1 @@ +Limit Soft Limit Hard Limit Units \ No newline at end of file diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/status b/vendor/github.com/VictoriaMetrics/metrics/testdata/status new file mode 100644 index 0000000..d67a15e --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/testdata/status @@ -0,0 +1,115 @@ +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080548 kB +VmSize: 2080464 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 105212 kB +RssAnon: 94092 kB +RssFile: 11120 kB +RssShmem: 0 kB +VmData: 632076 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 940 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 +/ # cat /proc/1/stat +stat statm status +/ # cat /proc/1/statm +520122 27057 2780 1751 0 158052 0 +/ # cat /proc/1/status +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080556 kB +VmSize: 2080520 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 129848 kB +RssAnon: 118752 kB +RssFile: 11096 kB +RssShmem: 0 kB +VmData: 633020 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 984 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 \ No newline at end of file diff --git a/vendor/github.com/VictoriaMetrics/metrics/testdata/status_bad b/vendor/github.com/VictoriaMetrics/metrics/testdata/status_bad new file mode 100644 index 0000000..24cf8cb --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/testdata/status_bad @@ -0,0 +1,115 @@ +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080548 kB +VmSize: 2080464 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 105212 kB +RssAnon: 94092 kB +RssFile: 11120 kB +RssShmem: 0 kB +VmData: 632076 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 940 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 +/ # cat /proc/1/stat +stat statm status +/ # cat /proc/1/statm +520122 27057 2780 1751 0 158052 0 +/ # cat /proc/1/status +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080556 kB +VmSize: 2080520 kB as +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 129848 kB +RssAnon: 118752 kB +RssFile: 11096 kB +RssShmem: 0 kB +VmData: 633020 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 984 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 fsa +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 \ No newline at end of file diff --git a/vendor/github.com/VictoriaMetrics/metrics/validator.go b/vendor/github.com/VictoriaMetrics/metrics/validator.go new file mode 100644 index 0000000..9960189 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/validator.go @@ -0,0 +1,84 @@ +package metrics + +import ( + "fmt" + "regexp" + "strings" +) + +func validateMetric(s string) error { + if len(s) == 0 { + return fmt.Errorf("metric cannot be empty") + } + n := strings.IndexByte(s, '{') + if n < 0 { + return validateIdent(s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[len(s)-1] != '}' { + return fmt.Errorf("missing closing curly brace at the end of %q", ident) + } + return validateTags(s[:len(s)-1]) +} + +func validateTags(s string) error { + if len(s) == 0 { + return nil + } + for { + n := strings.IndexByte(s, '=') + if n < 0 { + return fmt.Errorf("missing `=` after %q", s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[0] != '"' { + return fmt.Errorf("missing starting `\"` for %q value; tail=%q", ident, s) + } + s = s[1:] + again: + n = strings.IndexByte(s, '"') + if n < 0 { + return fmt.Errorf("missing trailing `\"` for %q value; tail=%q", ident, s) + } + m := n + for m > 0 && s[m-1] == '\\' { + m-- + } + if (n-m)%2 == 1 { + s = s[n+1:] + goto again + } + s = s[n+1:] + if len(s) == 0 { + return nil + } + if !strings.HasPrefix(s, ",") { + return fmt.Errorf("missing `,` after %q value; tail=%q", ident, s) + } + s = skipSpace(s[1:]) + } +} + +func skipSpace(s string) string { + for len(s) > 0 && s[0] == ' ' { + s = s[1:] + } + return s +} + +func validateIdent(s string) error { + if !identRegexp.MatchString(s) { + return fmt.Errorf("invalid identifier %q", s) + } + return nil +} + +var identRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$") diff --git a/vendor/github.com/VictoriaMetrics/metrics/validator_test.go b/vendor/github.com/VictoriaMetrics/metrics/validator_test.go new file mode 100644 index 0000000..8a5d734 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/validator_test.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "testing" +) + +func TestValidateMetricSuccess(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err != nil { + t.Fatalf("cannot validate %q: %s", s, err) + } + } + f("a") + f("_9:8") + f("a{}") + f(`a{foo="bar"}`) + f(`foo{bar="baz", x="y\"z"}`) + f(`foo{bar="b}az"}`) + f(`:foo:bar{bar="a",baz="b"}`) + f(`some.foo{bar="baz"}`) +} + +func TestValidateMetricError(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err == nil { + t.Fatalf("expecting non-nil error when validating %q", s) + } + } + f("") + f("{}") + + // superflouos space + f("a ") + f(" a") + f(" a ") + f("a {}") + f("a{} ") + f("a{ }") + f(`a{foo ="bar"}`) + f(`a{ foo="bar"}`) + f(`a{foo= "bar"}`) + f(`a{foo="bar" }`) + f(`a{foo="bar" ,baz="a"}`) + + // invalid tags + f("a{foo}") + f("a{=}") + f(`a{=""}`) + f(`a{`) + f(`a}`) + f(`a{foo=}`) + f(`a{foo="`) + f(`a{foo="}`) + f(`a{foo="bar",}`) + f(`a{foo="bar", x`) + f(`a{foo="bar", x=`) + f(`a{foo="bar", x="`) + f(`a{foo="bar", x="}`) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/.travis.yml b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/.travis.yml new file mode 100644 index 0000000..336ccde --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - 1.7 + - 1.8 + +script: + # build test for supported platforms + - GOOS=linux go build + - GOOS=darwin go build + - GOOS=freebsd go build + - GOARCH=386 go build + + # run tests on a standard platform + - go test -v ./... + diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/LICENSE b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/LICENSE new file mode 100644 index 0000000..a2b05f6 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/README.md b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/README.md new file mode 100644 index 0000000..3d384c6 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/README.md @@ -0,0 +1,76 @@ +[![Build Status](https://travis-ci.org/valyala/fastrand.svg)](https://travis-ci.org/valyala/fastrand) +[![GoDoc](https://godoc.org/github.com/valyala/fastrand?status.svg)](http://godoc.org/github.com/valyala/fastrand) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/fastrand)](https://goreportcard.com/report/github.com/valyala/fastrand) + + +# fastrand + +Fast pseudorandom number generator. + + +# Features + +- Optimized for speed. +- Performance scales on multiple CPUs. + +# How does it work? + +It abuses [sync.Pool](https://golang.org/pkg/sync/#Pool) for maintaining +"per-CPU" pseudorandom number generators. + +TODO: firgure out how to use real per-CPU pseudorandom number generators. + + +# Benchmark results + + +``` +$ GOMAXPROCS=1 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n 50000000 29.7 ns/op +BenchmarkRNGUint32n 200000000 6.50 ns/op +BenchmarkRNGUint32nWithLock 100000000 21.5 ns/op +BenchmarkMathRandInt31n 50000000 31.8 ns/op +BenchmarkMathRandRNGInt31n 100000000 17.9 ns/op +BenchmarkMathRandRNGInt31nWithLock 50000000 30.2 ns/op +PASS +ok github.com/valyala/fastrand 10.634s +``` + +``` +$ GOMAXPROCS=2 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-2 100000000 17.6 ns/op +BenchmarkRNGUint32n-2 500000000 3.36 ns/op +BenchmarkRNGUint32nWithLock-2 50000000 32.0 ns/op +BenchmarkMathRandInt31n-2 20000000 51.2 ns/op +BenchmarkMathRandRNGInt31n-2 100000000 11.0 ns/op +BenchmarkMathRandRNGInt31nWithLock-2 20000000 91.0 ns/op +PASS +ok github.com/valyala/fastrand 9.543s +``` + +``` +$ GOMAXPROCS=4 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-4 100000000 14.2 ns/op +BenchmarkRNGUint32n-4 500000000 3.30 ns/op +BenchmarkRNGUint32nWithLock-4 20000000 88.7 ns/op +BenchmarkMathRandInt31n-4 10000000 145 ns/op +BenchmarkMathRandRNGInt31n-4 200000000 8.35 ns/op +BenchmarkMathRandRNGInt31nWithLock-4 20000000 102 ns/op +PASS +ok github.com/valyala/fastrand 11.534s +``` + +As you can see, [fastrand.Uint32n](https://godoc.org/github.com/valyala/fastrand#Uint32n) +scales on multiple CPUs, while [rand.Int31n](https://golang.org/pkg/math/rand/#Int31n) +doesn't scale. Their performance is comparable on `GOMAXPROCS=1`, +but `fastrand.Uint32n` runs 3x faster than `rand.Int31n` on `GOMAXPROCS=2` +and 10x faster than `rand.Int31n` on `GOMAXPROCS=4`. diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/fastrand.go b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/fastrand.go new file mode 100644 index 0000000..3ea9177 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/fastrand.go @@ -0,0 +1,74 @@ +// Package fastrand implements fast pesudorandom number generator +// that should scale well on multi-CPU systems. +// +// Use crypto/rand instead of this package for generating +// cryptographically secure random numbers. +package fastrand + +import ( + "sync" + "time" +) + +// Uint32 returns pseudorandom uint32. +// +// It is safe calling this function from concurrent goroutines. +func Uint32() uint32 { + v := rngPool.Get() + if v == nil { + v = &RNG{} + } + r := v.(*RNG) + x := r.Uint32() + rngPool.Put(r) + return x +} + +var rngPool sync.Pool + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is safe calling this function from concurrent goroutines. +func Uint32n(maxN uint32) uint32 { + x := Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +// RNG is a pseudorandom number generator. +// +// It is unsafe to call RNG methods from concurrent goroutines. +type RNG struct { + x uint32 +} + +// Uint32 returns pseudorandom uint32. +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32() uint32 { + for r.x == 0 { + r.x = getRandomUint32() + } + + // See https://en.wikipedia.org/wiki/Xorshift + x := r.x + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + r.x = x + return x +} + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32n(maxN uint32) uint32 { + x := r.Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +func getRandomUint32() uint32 { + x := time.Now().UnixNano() + return uint32((x >> 32) ^ x) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/go.mod b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/go.mod new file mode 100644 index 0000000..958910b --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/fastrand/go.mod @@ -0,0 +1 @@ +module github.com/valyala/fastrand diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/LICENSE b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/LICENSE new file mode 100644 index 0000000..902bcad --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/README.md b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/README.md new file mode 100644 index 0000000..6d4eb59 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/README.md @@ -0,0 +1,9 @@ +[![GoDoc](https://godoc.org/github.com/valyala/histogram?status.svg)](http://godoc.org/github.com/valyala/histogram) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/histogram)](https://goreportcard.com/report/github.com/valyala/histogram) + + +# histogram + +Fast histograms for Go. + +See [docs](https://godoc.org/github.com/valyala/histogram). diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.mod b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.mod new file mode 100644 index 0000000..984efbe --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.mod @@ -0,0 +1,5 @@ +module github.com/valyala/histogram + +go 1.12 + +require github.com/valyala/fastrand v1.0.0 diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.sum b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.sum new file mode 100644 index 0000000..2b3e848 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/go.sum @@ -0,0 +1,2 @@ +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/histogram.go b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/histogram.go new file mode 100644 index 0000000..e90dd70 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/github.com/valyala/histogram/histogram.go @@ -0,0 +1,127 @@ +// Package histogram provides building blocks for fast histograms. +package histogram + +import ( + "math" + "sort" + "sync" + + "github.com/valyala/fastrand" +) + +var ( + infNeg = math.Inf(-1) + infPos = math.Inf(1) + nan = math.NaN() +) + +// Fast is a fast histogram. +// +// It cannot be used from concurrently running goroutines without +// external synchronization. +type Fast struct { + max float64 + min float64 + count uint64 + + a []float64 + tmp []float64 + rng fastrand.RNG +} + +// NewFast returns new fast histogram. +func NewFast() *Fast { + f := &Fast{} + f.Reset() + return f +} + +// Reset resets the histogram. +func (f *Fast) Reset() { + f.max = infNeg + f.min = infPos + f.count = 0 + if len(f.a) > 0 { + f.a = f.a[:0] + f.tmp = f.tmp[:0] + } else { + // Free up memory occupied by unused histogram. + f.a = nil + f.tmp = nil + } +} + +// Update updates the f with v. +func (f *Fast) Update(v float64) { + if v > f.max { + f.max = v + } + if v < f.min { + f.min = v + } + + f.count++ + if len(f.a) < maxSamples { + f.a = append(f.a, v) + return + } + if n := int(f.rng.Uint32n(uint32(f.count))); n < len(f.a) { + f.a[n] = v + } +} + +const maxSamples = 1000 + +// Quantile returns the quantile value for the given phi. +func (f *Fast) Quantile(phi float64) float64 { + f.tmp = append(f.tmp[:0], f.a...) + sort.Float64s(f.tmp) + return f.quantile(phi) +} + +// Quantiles appends quantile values to dst for the given phis. +func (f *Fast) Quantiles(dst, phis []float64) []float64 { + f.tmp = append(f.tmp[:0], f.a...) + sort.Float64s(f.tmp) + for _, phi := range phis { + q := f.quantile(phi) + dst = append(dst, q) + } + return dst +} + +func (f *Fast) quantile(phi float64) float64 { + if len(f.tmp) == 0 || math.IsNaN(phi) { + return nan + } + if phi <= 0 { + return f.min + } + if phi >= 1 { + return f.max + } + idx := uint(phi*float64(len(f.tmp)-1) + 0.5) + if idx >= uint(len(f.tmp)) { + idx = uint(len(f.tmp) - 1) + } + return f.tmp[idx] +} + +// GetFast returns a histogram from a pool. +func GetFast() *Fast { + v := fastPool.Get() + if v == nil { + return NewFast() + } + return v.(*Fast) +} + +// PutFast puts hf to the pool. +// +// hf cannot be used after this call. +func PutFast(f *Fast) { + f.Reset() + fastPool.Put(f) +} + +var fastPool sync.Pool diff --git a/vendor/github.com/VictoriaMetrics/metrics/vendor/modules.txt b/vendor/github.com/VictoriaMetrics/metrics/vendor/modules.txt new file mode 100644 index 0000000..f915051 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/vendor/modules.txt @@ -0,0 +1,4 @@ +# github.com/valyala/fastrand v1.0.0 +github.com/valyala/fastrand +# github.com/valyala/histogram v1.1.2 +github.com/valyala/histogram diff --git a/vendor/github.com/valyala/fastrand/.travis.yml b/vendor/github.com/valyala/fastrand/.travis.yml new file mode 100644 index 0000000..336ccde --- /dev/null +++ b/vendor/github.com/valyala/fastrand/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - 1.7 + - 1.8 + +script: + # build test for supported platforms + - GOOS=linux go build + - GOOS=darwin go build + - GOOS=freebsd go build + - GOARCH=386 go build + + # run tests on a standard platform + - go test -v ./... + diff --git a/vendor/github.com/valyala/fastrand/LICENSE b/vendor/github.com/valyala/fastrand/LICENSE new file mode 100644 index 0000000..a2b05f6 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/valyala/fastrand/README.md b/vendor/github.com/valyala/fastrand/README.md new file mode 100644 index 0000000..3d384c6 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/README.md @@ -0,0 +1,76 @@ +[![Build Status](https://travis-ci.org/valyala/fastrand.svg)](https://travis-ci.org/valyala/fastrand) +[![GoDoc](https://godoc.org/github.com/valyala/fastrand?status.svg)](http://godoc.org/github.com/valyala/fastrand) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/fastrand)](https://goreportcard.com/report/github.com/valyala/fastrand) + + +# fastrand + +Fast pseudorandom number generator. + + +# Features + +- Optimized for speed. +- Performance scales on multiple CPUs. + +# How does it work? + +It abuses [sync.Pool](https://golang.org/pkg/sync/#Pool) for maintaining +"per-CPU" pseudorandom number generators. + +TODO: firgure out how to use real per-CPU pseudorandom number generators. + + +# Benchmark results + + +``` +$ GOMAXPROCS=1 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n 50000000 29.7 ns/op +BenchmarkRNGUint32n 200000000 6.50 ns/op +BenchmarkRNGUint32nWithLock 100000000 21.5 ns/op +BenchmarkMathRandInt31n 50000000 31.8 ns/op +BenchmarkMathRandRNGInt31n 100000000 17.9 ns/op +BenchmarkMathRandRNGInt31nWithLock 50000000 30.2 ns/op +PASS +ok github.com/valyala/fastrand 10.634s +``` + +``` +$ GOMAXPROCS=2 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-2 100000000 17.6 ns/op +BenchmarkRNGUint32n-2 500000000 3.36 ns/op +BenchmarkRNGUint32nWithLock-2 50000000 32.0 ns/op +BenchmarkMathRandInt31n-2 20000000 51.2 ns/op +BenchmarkMathRandRNGInt31n-2 100000000 11.0 ns/op +BenchmarkMathRandRNGInt31nWithLock-2 20000000 91.0 ns/op +PASS +ok github.com/valyala/fastrand 9.543s +``` + +``` +$ GOMAXPROCS=4 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-4 100000000 14.2 ns/op +BenchmarkRNGUint32n-4 500000000 3.30 ns/op +BenchmarkRNGUint32nWithLock-4 20000000 88.7 ns/op +BenchmarkMathRandInt31n-4 10000000 145 ns/op +BenchmarkMathRandRNGInt31n-4 200000000 8.35 ns/op +BenchmarkMathRandRNGInt31nWithLock-4 20000000 102 ns/op +PASS +ok github.com/valyala/fastrand 11.534s +``` + +As you can see, [fastrand.Uint32n](https://godoc.org/github.com/valyala/fastrand#Uint32n) +scales on multiple CPUs, while [rand.Int31n](https://golang.org/pkg/math/rand/#Int31n) +doesn't scale. Their performance is comparable on `GOMAXPROCS=1`, +but `fastrand.Uint32n` runs 3x faster than `rand.Int31n` on `GOMAXPROCS=2` +and 10x faster than `rand.Int31n` on `GOMAXPROCS=4`. diff --git a/vendor/github.com/valyala/fastrand/fastrand.go b/vendor/github.com/valyala/fastrand/fastrand.go new file mode 100644 index 0000000..3ea9177 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/fastrand.go @@ -0,0 +1,74 @@ +// Package fastrand implements fast pesudorandom number generator +// that should scale well on multi-CPU systems. +// +// Use crypto/rand instead of this package for generating +// cryptographically secure random numbers. +package fastrand + +import ( + "sync" + "time" +) + +// Uint32 returns pseudorandom uint32. +// +// It is safe calling this function from concurrent goroutines. +func Uint32() uint32 { + v := rngPool.Get() + if v == nil { + v = &RNG{} + } + r := v.(*RNG) + x := r.Uint32() + rngPool.Put(r) + return x +} + +var rngPool sync.Pool + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is safe calling this function from concurrent goroutines. +func Uint32n(maxN uint32) uint32 { + x := Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +// RNG is a pseudorandom number generator. +// +// It is unsafe to call RNG methods from concurrent goroutines. +type RNG struct { + x uint32 +} + +// Uint32 returns pseudorandom uint32. +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32() uint32 { + for r.x == 0 { + r.x = getRandomUint32() + } + + // See https://en.wikipedia.org/wiki/Xorshift + x := r.x + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + r.x = x + return x +} + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32n(maxN uint32) uint32 { + x := r.Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +func getRandomUint32() uint32 { + x := time.Now().UnixNano() + return uint32((x >> 32) ^ x) +} diff --git a/vendor/github.com/valyala/fastrand/go.mod b/vendor/github.com/valyala/fastrand/go.mod new file mode 100644 index 0000000..958910b --- /dev/null +++ b/vendor/github.com/valyala/fastrand/go.mod @@ -0,0 +1 @@ +module github.com/valyala/fastrand diff --git a/vendor/github.com/valyala/histogram/LICENSE b/vendor/github.com/valyala/histogram/LICENSE new file mode 100644 index 0000000..902bcad --- /dev/null +++ b/vendor/github.com/valyala/histogram/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/valyala/histogram/README.md b/vendor/github.com/valyala/histogram/README.md new file mode 100644 index 0000000..6d4eb59 --- /dev/null +++ b/vendor/github.com/valyala/histogram/README.md @@ -0,0 +1,9 @@ +[![GoDoc](https://godoc.org/github.com/valyala/histogram?status.svg)](http://godoc.org/github.com/valyala/histogram) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/histogram)](https://goreportcard.com/report/github.com/valyala/histogram) + + +# histogram + +Fast histograms for Go. + +See [docs](https://godoc.org/github.com/valyala/histogram). diff --git a/vendor/github.com/valyala/histogram/go.mod b/vendor/github.com/valyala/histogram/go.mod new file mode 100644 index 0000000..984efbe --- /dev/null +++ b/vendor/github.com/valyala/histogram/go.mod @@ -0,0 +1,5 @@ +module github.com/valyala/histogram + +go 1.12 + +require github.com/valyala/fastrand v1.0.0 diff --git a/vendor/github.com/valyala/histogram/go.sum b/vendor/github.com/valyala/histogram/go.sum new file mode 100644 index 0000000..2b3e848 --- /dev/null +++ b/vendor/github.com/valyala/histogram/go.sum @@ -0,0 +1,2 @@ +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= diff --git a/vendor/github.com/valyala/histogram/histogram.go b/vendor/github.com/valyala/histogram/histogram.go new file mode 100644 index 0000000..e90dd70 --- /dev/null +++ b/vendor/github.com/valyala/histogram/histogram.go @@ -0,0 +1,127 @@ +// Package histogram provides building blocks for fast histograms. +package histogram + +import ( + "math" + "sort" + "sync" + + "github.com/valyala/fastrand" +) + +var ( + infNeg = math.Inf(-1) + infPos = math.Inf(1) + nan = math.NaN() +) + +// Fast is a fast histogram. +// +// It cannot be used from concurrently running goroutines without +// external synchronization. +type Fast struct { + max float64 + min float64 + count uint64 + + a []float64 + tmp []float64 + rng fastrand.RNG +} + +// NewFast returns new fast histogram. +func NewFast() *Fast { + f := &Fast{} + f.Reset() + return f +} + +// Reset resets the histogram. +func (f *Fast) Reset() { + f.max = infNeg + f.min = infPos + f.count = 0 + if len(f.a) > 0 { + f.a = f.a[:0] + f.tmp = f.tmp[:0] + } else { + // Free up memory occupied by unused histogram. + f.a = nil + f.tmp = nil + } +} + +// Update updates the f with v. +func (f *Fast) Update(v float64) { + if v > f.max { + f.max = v + } + if v < f.min { + f.min = v + } + + f.count++ + if len(f.a) < maxSamples { + f.a = append(f.a, v) + return + } + if n := int(f.rng.Uint32n(uint32(f.count))); n < len(f.a) { + f.a[n] = v + } +} + +const maxSamples = 1000 + +// Quantile returns the quantile value for the given phi. +func (f *Fast) Quantile(phi float64) float64 { + f.tmp = append(f.tmp[:0], f.a...) + sort.Float64s(f.tmp) + return f.quantile(phi) +} + +// Quantiles appends quantile values to dst for the given phis. +func (f *Fast) Quantiles(dst, phis []float64) []float64 { + f.tmp = append(f.tmp[:0], f.a...) + sort.Float64s(f.tmp) + for _, phi := range phis { + q := f.quantile(phi) + dst = append(dst, q) + } + return dst +} + +func (f *Fast) quantile(phi float64) float64 { + if len(f.tmp) == 0 || math.IsNaN(phi) { + return nan + } + if phi <= 0 { + return f.min + } + if phi >= 1 { + return f.max + } + idx := uint(phi*float64(len(f.tmp)-1) + 0.5) + if idx >= uint(len(f.tmp)) { + idx = uint(len(f.tmp) - 1) + } + return f.tmp[idx] +} + +// GetFast returns a histogram from a pool. +func GetFast() *Fast { + v := fastPool.Get() + if v == nil { + return NewFast() + } + return v.(*Fast) +} + +// PutFast puts hf to the pool. +// +// hf cannot be used after this call. +func PutFast(f *Fast) { + f.Reset() + fastPool.Put(f) +} + +var fastPool sync.Pool diff --git a/vendor/github.com/valyala/histogram/histogram_test.go b/vendor/github.com/valyala/histogram/histogram_test.go new file mode 100644 index 0000000..d1b5a12 --- /dev/null +++ b/vendor/github.com/valyala/histogram/histogram_test.go @@ -0,0 +1,59 @@ +package histogram + +import ( + "math" + "testing" +) + +func TestFastUnderflow(t *testing.T) { + f := GetFast() + defer PutFast(f) + + q := f.Quantile(0.5) + if !math.IsNaN(q) { + t.Fatalf("unexpected quantile for empty histogram; got %v; want %v", q, nan) + } + + for i := 0; i < maxSamples; i++ { + f.Update(float64(i)) + } + qs := f.Quantiles(nil, []float64{0, 0.5, 1}) + if qs[0] != 0 { + t.Fatalf("unexpected quantile value for phi=0; got %v; want %v", qs[0], 0) + } + if qs[1] != maxSamples/2 { + t.Fatalf("unexpected quantile value for phi=0.5; got %v; want %v", qs[1], maxSamples/2) + } + if qs[2] != maxSamples-1 { + t.Fatalf("unexpected quantile value for phi=1; got %v; want %v", qs[2], maxSamples-1) + } +} + +func TestFastOverflow(t *testing.T) { + f := GetFast() + defer PutFast(f) + + for i := 0; i < maxSamples*10; i++ { + f.Update(float64(i)) + } + qs := f.Quantiles(nil, []float64{0, 0.5, 0.9999, 1}) + if qs[0] != 0 { + t.Fatalf("unexpected quantile value for phi=0; got %v; want %v", qs[0], 0) + } + + median := float64(maxSamples*10-1) / 2 + if qs[1] < median*0.9 || qs[1] > median*1.1 { + t.Fatalf("unexpected quantile value for phi=0.5; got %v; want %v", qs[1], median) + } + if qs[2] < maxSamples*10*0.9 { + t.Fatalf("unexpected quantile value for phi=0.9999; got %v; want %v", qs[2], maxSamples*10*0.9) + } + if qs[3] != maxSamples*10-1 { + t.Fatalf("unexpected quantile value for phi=1; got %v; want %v", qs[3], maxSamples*10-1) + } + + q := f.Quantile(nan) + if !math.IsNaN(q) { + t.Fatalf("unexpected value for phi=NaN; got %v; want %v", q, nan) + } +} diff --git a/vendor/github.com/valyala/histogram/histogram_timing_test.go b/vendor/github.com/valyala/histogram/histogram_timing_test.go new file mode 100644 index 0000000..a52caae --- /dev/null +++ b/vendor/github.com/valyala/histogram/histogram_timing_test.go @@ -0,0 +1,25 @@ +package histogram + +import ( + "sync" + "testing" +) + +func BenchmarkFastUpdate(b *testing.B) { + b.ReportAllocs() + b.SetBytes(1) + b.RunParallel(func(pb *testing.PB) { + f := NewFast() + var v float64 + for pb.Next() { + f.Update(v) + v += 1.5 + } + SinkLock.Lock() + Sink += f.Quantile(0.5) + SinkLock.Unlock() + }) +} + +var Sink float64 +var SinkLock sync.Mutex diff --git a/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/.travis.yml b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/.travis.yml new file mode 100644 index 0000000..336ccde --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - 1.7 + - 1.8 + +script: + # build test for supported platforms + - GOOS=linux go build + - GOOS=darwin go build + - GOOS=freebsd go build + - GOARCH=386 go build + + # run tests on a standard platform + - go test -v ./... + diff --git a/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/LICENSE b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/LICENSE new file mode 100644 index 0000000..a2b05f6 --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/README.md b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/README.md new file mode 100644 index 0000000..3d384c6 --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/README.md @@ -0,0 +1,76 @@ +[![Build Status](https://travis-ci.org/valyala/fastrand.svg)](https://travis-ci.org/valyala/fastrand) +[![GoDoc](https://godoc.org/github.com/valyala/fastrand?status.svg)](http://godoc.org/github.com/valyala/fastrand) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/fastrand)](https://goreportcard.com/report/github.com/valyala/fastrand) + + +# fastrand + +Fast pseudorandom number generator. + + +# Features + +- Optimized for speed. +- Performance scales on multiple CPUs. + +# How does it work? + +It abuses [sync.Pool](https://golang.org/pkg/sync/#Pool) for maintaining +"per-CPU" pseudorandom number generators. + +TODO: firgure out how to use real per-CPU pseudorandom number generators. + + +# Benchmark results + + +``` +$ GOMAXPROCS=1 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n 50000000 29.7 ns/op +BenchmarkRNGUint32n 200000000 6.50 ns/op +BenchmarkRNGUint32nWithLock 100000000 21.5 ns/op +BenchmarkMathRandInt31n 50000000 31.8 ns/op +BenchmarkMathRandRNGInt31n 100000000 17.9 ns/op +BenchmarkMathRandRNGInt31nWithLock 50000000 30.2 ns/op +PASS +ok github.com/valyala/fastrand 10.634s +``` + +``` +$ GOMAXPROCS=2 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-2 100000000 17.6 ns/op +BenchmarkRNGUint32n-2 500000000 3.36 ns/op +BenchmarkRNGUint32nWithLock-2 50000000 32.0 ns/op +BenchmarkMathRandInt31n-2 20000000 51.2 ns/op +BenchmarkMathRandRNGInt31n-2 100000000 11.0 ns/op +BenchmarkMathRandRNGInt31nWithLock-2 20000000 91.0 ns/op +PASS +ok github.com/valyala/fastrand 9.543s +``` + +``` +$ GOMAXPROCS=4 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-4 100000000 14.2 ns/op +BenchmarkRNGUint32n-4 500000000 3.30 ns/op +BenchmarkRNGUint32nWithLock-4 20000000 88.7 ns/op +BenchmarkMathRandInt31n-4 10000000 145 ns/op +BenchmarkMathRandRNGInt31n-4 200000000 8.35 ns/op +BenchmarkMathRandRNGInt31nWithLock-4 20000000 102 ns/op +PASS +ok github.com/valyala/fastrand 11.534s +``` + +As you can see, [fastrand.Uint32n](https://godoc.org/github.com/valyala/fastrand#Uint32n) +scales on multiple CPUs, while [rand.Int31n](https://golang.org/pkg/math/rand/#Int31n) +doesn't scale. Their performance is comparable on `GOMAXPROCS=1`, +but `fastrand.Uint32n` runs 3x faster than `rand.Int31n` on `GOMAXPROCS=2` +and 10x faster than `rand.Int31n` on `GOMAXPROCS=4`. diff --git a/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/fastrand.go b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/fastrand.go new file mode 100644 index 0000000..3ea9177 --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/fastrand.go @@ -0,0 +1,74 @@ +// Package fastrand implements fast pesudorandom number generator +// that should scale well on multi-CPU systems. +// +// Use crypto/rand instead of this package for generating +// cryptographically secure random numbers. +package fastrand + +import ( + "sync" + "time" +) + +// Uint32 returns pseudorandom uint32. +// +// It is safe calling this function from concurrent goroutines. +func Uint32() uint32 { + v := rngPool.Get() + if v == nil { + v = &RNG{} + } + r := v.(*RNG) + x := r.Uint32() + rngPool.Put(r) + return x +} + +var rngPool sync.Pool + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is safe calling this function from concurrent goroutines. +func Uint32n(maxN uint32) uint32 { + x := Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +// RNG is a pseudorandom number generator. +// +// It is unsafe to call RNG methods from concurrent goroutines. +type RNG struct { + x uint32 +} + +// Uint32 returns pseudorandom uint32. +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32() uint32 { + for r.x == 0 { + r.x = getRandomUint32() + } + + // See https://en.wikipedia.org/wiki/Xorshift + x := r.x + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + r.x = x + return x +} + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32n(maxN uint32) uint32 { + x := r.Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +func getRandomUint32() uint32 { + x := time.Now().UnixNano() + return uint32((x >> 32) ^ x) +} diff --git a/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/go.mod b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/go.mod new file mode 100644 index 0000000..958910b --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/github.com/valyala/fastrand/go.mod @@ -0,0 +1 @@ +module github.com/valyala/fastrand diff --git a/vendor/github.com/valyala/histogram/vendor/modules.txt b/vendor/github.com/valyala/histogram/vendor/modules.txt new file mode 100644 index 0000000..9871e30 --- /dev/null +++ b/vendor/github.com/valyala/histogram/vendor/modules.txt @@ -0,0 +1,2 @@ +# github.com/valyala/fastrand v1.0.0 +github.com/valyala/fastrand diff --git a/walletapi/daemon_communication.go b/walletapi/daemon_communication.go index 3e75fcc..9dde1f2 100644 --- a/walletapi/daemon_communication.go +++ b/walletapi/daemon_communication.go @@ -157,7 +157,7 @@ func (w *Wallet_Memory) Sync_Wallet_Memory_With_Daemon() { rlog.Debugf("wallet topo height %d daemon online topo height %d\n", w.account.TopoHeight, w.Daemon_TopoHeight) previous := w.account.Balance_Result.Data var scid crypto.Hash - if _, _, err := w.GetEncryptedBalanceAtTopoHeight(scid, -1, w.GetAddress().String()); err == nil { + if _, _,_,_,_, err := w.GetEncryptedBalanceAtTopoHeight(scid, -1, w.GetAddress().String()); err == nil { if w.account.Balance_Result.Data != previous /*|| (len(w.account.EntriesNative[scid]) >= 1 && strings.ToLower(w.account.Balance_Result.Data) != strings.ToLower(w.account.EntriesNative[scid][len(w.account.EntriesNative[scid])-1].EWData)) */ { w.DecodeEncryptedBalance() // try to decode balance @@ -240,7 +240,7 @@ func (w *Wallet_Memory) DecodeEncryptedBalanceNow(el *crypto.ElGamal) uint64 { // TODO in order to stop privacy leaks we must guess this information somehow on client side itself // maybe the server can broadcast a bloomfilter or something else from the mempool keyimages // -func (w *Wallet_Memory) GetEncryptedBalanceAtTopoHeight(scid crypto.Hash, topoheight int64, accountaddr string) (bits int, e *crypto.ElGamal, err error) { +func (w *Wallet_Memory) GetEncryptedBalanceAtTopoHeight(scid crypto.Hash, topoheight int64, accountaddr string) (bits int, e *crypto.ElGamal, height,rtopoheight uint64, merkleroot crypto.Hash, err error) { defer func() { if r := recover(); r != nil { @@ -300,7 +300,7 @@ func (w *Wallet_Memory) GetEncryptedBalanceAtTopoHeight(scid crypto.Hash, topohe } } - // fmt.Printf("status '%s' err '%s' %+v %+v \n", result.Status , w.Error , result.Status == errormsg.ErrAccountUnregistered.Error() , accountaddr == w.account.GetAddress().String()) +// fmt.Printf("status '%s' err '%s' %+v %+v \n", result.Status , w.Error , result.Status == errormsg.ErrAccountUnregistered.Error() , accountaddr == w.account.GetAddress().String()) if scid.IsZero() && result.Status == errormsg.ErrAccountUnregistered.Error() { err = fmt.Errorf("%s", result.Status) @@ -311,7 +311,7 @@ func (w *Wallet_Memory) GetEncryptedBalanceAtTopoHeight(scid crypto.Hash, topohe w.Daemon_TopoHeight = result.DTopoheight w.Merkle_Balance_TreeHash = result.DMerkle_Balance_TreeHash - if scid.IsZero() && accountaddr == w.GetAddress().String() { + if topoheight == -1 && scid.IsZero() && accountaddr == w.GetAddress().String() { w.account.Balance_Result = result w.account.TopoHeight = result.Topoheight } @@ -330,10 +330,17 @@ func (w *Wallet_Memory) GetEncryptedBalanceAtTopoHeight(scid crypto.Hash, topohe w.Error = nil } - //fmt.Printf("decoding elgamal\n") - el := new(crypto.ElGamal).Deserialize(hexdecoded) - return result.Bits, el, nil + + hexdecoded, err = hex.DecodeString(result.Merkle_Balance_TreeHash) + if err != nil { + return + } + + var mhash crypto.Hash + copy(mhash[:],hexdecoded[:]) + + return result.Bits, el, uint64(result.Height), uint64(result.Topoheight), mhash, nil } func (w *Wallet_Memory) DecodeEncryptedBalance_Memory(el *crypto.ElGamal, hint uint64) (balance uint64) { @@ -346,7 +353,7 @@ func (w *Wallet_Memory) DecodeEncryptedBalance_Memory(el *crypto.ElGamal, hint u } func (w *Wallet_Memory) GetDecryptedBalanceAtTopoHeight(scid crypto.Hash, topoheight int64, accountaddr string) (balance uint64, err error) { - _, encrypted_balance, err := w.GetEncryptedBalanceAtTopoHeight(scid, topoheight, accountaddr) + _, encrypted_balance,_,_,_, err := w.GetEncryptedBalanceAtTopoHeight(scid, topoheight, accountaddr) if err != nil { return 0, err } @@ -466,13 +473,13 @@ func (w *Wallet_Memory) synchistory_internal(scid crypto.Hash, start_topo, end_t if start_topo == w.account.Balance_Result.Registration { start_balance_e = crypto.ConstructElGamal(w.account.Keys.Public.G1(), crypto.ElGamal_BASE_G) } else { - _, start_balance_e, err = w.GetEncryptedBalanceAtTopoHeight(scid, start_topo, w.GetAddress().String()) + _, start_balance_e,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(scid, start_topo, w.GetAddress().String()) if err != nil { return err } } - _, end_balance_e, err := w.GetEncryptedBalanceAtTopoHeight(scid, end_topo, w.GetAddress().String()) + _, end_balance_e,_,_,_, err := w.GetEncryptedBalanceAtTopoHeight(scid, end_topo, w.GetAddress().String()) if err != nil { return err } @@ -512,7 +519,7 @@ func (w *Wallet_Memory) synchistory_internal_binary_search(scid crypto.Hash, sta return w.synchistory_block(scid, end_topo) } - _, median_balance_e, err := w.GetEncryptedBalanceAtTopoHeight(scid, median, w.GetAddress().String()) + _, median_balance_e,_,_,_, err := w.GetEncryptedBalanceAtTopoHeight(scid, median, w.GetAddress().String()) if err != nil { return err } @@ -599,13 +606,13 @@ func (w *Wallet_Memory) synchistory_block(scid crypto.Hash, topo int64) (err err if topo <= 0 || w.account.Balance_Result.Registration == topo { previous_balance_e = crypto.ConstructElGamal(w.account.Keys.Public.G1(), crypto.ElGamal_BASE_G) } else { - _, previous_balance_e, err = w.GetEncryptedBalanceAtTopoHeight(scid, topo-1, w.GetAddress().String()) + _, previous_balance_e,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(scid, topo-1, w.GetAddress().String()) if err != nil { return err } } - _, current_balance_e, err = w.GetEncryptedBalanceAtTopoHeight(scid, topo, w.GetAddress().String()) + _, current_balance_e,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(scid, topo, w.GetAddress().String()) if err != nil { return err } @@ -627,6 +634,10 @@ func (w *Wallet_Memory) synchistory_block(scid crypto.Hash, topo int64) (err err return fmt.Errorf("getblock rpc failed") } + if bresult.Block_Header.SideBlock { // skip side blocks + return nil + } + block_bin, _ := hex.DecodeString(bresult.Blob) bl.Deserialize(block_bin) diff --git a/walletapi/transaction_build.go b/walletapi/transaction_build.go index 1a05f0f..dfa1f6b 100644 --- a/walletapi/transaction_build.go +++ b/walletapi/transaction_build.go @@ -2,16 +2,16 @@ package walletapi import "fmt" import "math/big" - -//import "encoding/binary" import mathrand "math/rand" -import "github.com/deroproject/derohe/globals" import "github.com/deroproject/derohe/rpc" +import "github.com/deroproject/derohe/globals" +import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/transaction" import "github.com/deroproject/derohe/cryptography/crypto" import "github.com/deroproject/derohe/cryptography/bn256" // generate proof etc +// we use a previous point in history and cryptographically prove that we have not used the funds till now func (w *Wallet_Memory) BuildTransaction(transfers []rpc.Transfer, emap map[string]map[string][]byte, rings [][]*bn256.G1, height uint64, scdata rpc.Arguments, roothash []byte, max_bits_array []int) *transaction.Transaction { var tx transaction.Transaction @@ -22,6 +22,11 @@ func (w *Wallet_Memory) BuildTransaction(transfers []rpc.Transfer, emap map[stri tx.Version = 1 tx.Height = height tx.TransactionType = transaction.NORMAL + + + if height % config.BLOCK_BATCH_SIZE != 0 { + panic(fmt.Sprintf("Height must be a multiple of %d (config.BLOCK_BATCH_SIZE)", config.BLOCK_BATCH_SIZE)) + } /* if burn_value >= 1 { tx.TransactionType = transaction.BURN_TX @@ -217,9 +222,12 @@ func (w *Wallet_Memory) BuildTransaction(transfers []rpc.Transfer, emap map[stri } - u := new(bn256.G1).ScalarMult(crypto.HashToPoint(crypto.HashtoNumber(append([]byte(crypto.PROTOCOL_CONSTANT), tx.Payloads[0].Statement.Roothash[:]...))), sender_secret) // this should be moved to generate proof + + + u := new(bn256.G1).ScalarMult(crypto.HeightToPoint(height), sender_secret) // this should be moved to generate proof + u1 := new(bn256.G1).ScalarMult(crypto.HeightToPoint(height + config.BLOCK_BATCH_SIZE), sender_secret) // this should be moved to generate proof for t := range transfers { - tx.Payloads[t].Proof = crypto.GenerateProof(&tx.Payloads[t].Statement, &witness_list[t], u, tx.GetHash(), tx.Payloads[t].BurnValue) + tx.Payloads[t].Proof = crypto.GenerateProof(&tx.Payloads[t].Statement, &witness_list[t], u,u1, height, tx.GetHash(), tx.Payloads[t].BurnValue) } // after the tx is serialized, it loses information which is then fed by blockchain @@ -227,8 +235,8 @@ func (w *Wallet_Memory) BuildTransaction(transfers []rpc.Transfer, emap map[stri //fmt.Printf("txhash before %s\n", tx.GetHash()) for t := range tx.Payloads { - if tx.Payloads[t].Proof.Verify(&tx.Payloads[t].Statement, tx.GetHash(), tx.Payloads[t].BurnValue) { - //fmt.Printf("TX verified with proof successfuly %s burn_value %d\n", tx.GetHash(), tx.Payloads[t].BurnValue) + if tx.Payloads[t].Proof.Verify(&tx.Payloads[t].Statement, tx.GetHash(), height, tx.Payloads[t].BurnValue) { + fmt.Printf("TX verified with proof successfuly %s burn_value %d\n", tx.GetHash(), tx.Payloads[t].BurnValue) //fmt.Printf("Statement %+v\n", tx.Payloads[t].Statement) //fmt.Printf("Proof %+v\n", tx.Payloads[t].Proof) diff --git a/walletapi/wallet_pool.go b/walletapi/wallet_pool.go index e8b3e73..0a9c23a 100644 --- a/walletapi/wallet_pool.go +++ b/walletapi/wallet_pool.go @@ -29,7 +29,7 @@ import "github.com/romana/rlog" //import "github.com/vmihailenco/msgpack" -//import "github.com/deroproject/derohe/config" +import "github.com/deroproject/derohe/config" import "github.com/deroproject/derohe/cryptography/crypto" //import "github.com/deroproject/derohe/crypto/ringct" @@ -121,6 +121,7 @@ func (w *Wallet_Memory) PoolTransfer(transfers []rpc.Transfer, scdata rpc.Argume entry.Transfer_Everything = transfer_all entry.Trigger_Height = int64(w.Daemon_Height) + defer w.processPool(false) w.account.Lock() defer w.account.Unlock() w.account.Pool = append(w.account.Pool, entry) @@ -219,9 +220,9 @@ func (w *Wallet_Memory) processPool(checkonly bool) error { return fmt.Errorf("gettransa rpc failed err %s", err) } - if tx_result.Txs_as_hex[0] == "" { + if tx_result.Txs_as_hex[0] == "" { // we need to work this code better try.Status = "Lost (not in mempool/chain), Waiting for more blocks to retry" - if try.Height < (info.StableHeight + 1) { // we have attempted, lets wait some blocks, this needs to be optimized, for instant transfer + if try.Height + 3*config.BLOCK_BATCH_SIZE < (info.StableHeight + 1) { // we have attempted, lets wait some blocks, this needs to be optimized, for instant transfer // we need to send this tx again now }else{ continue // try other txs @@ -229,18 +230,27 @@ func (w *Wallet_Memory) processPool(checkonly bool) error { } else if tx_result.Txs[0].In_pool { try.Status = "TX in Mempool" continue - } else if tx_result.Txs[0].ValidBlock != "" { // if the result is valid in one of the blocks - try.Status = fmt.Sprintf("Mined in %s (%d confirmations)", tx_result.Txs[0].ValidBlock, info.TopoHeight-tx_result.Txs[0].Block_Height) - if try.Height < (info.StableHeight + 1) { // successful confirmation + } else if len(tx_result.Txs[0].MinedBlock) >= 1 { // if the result tx has been mined in one of the blocks + + + var bl_result rpc.GetBlockHeaderByHeight_Result + // Issue a call with a response. + if err := rpc_client.Call("DERO.GetBlockHeaderByHash", rpc.GetBlockHeaderByHash_Params{Hash: tx_result.Txs[0].MinedBlock[0]}, &bl_result); err != nil { + rlog.Errorf("GetBlockHeaderByTopoHeight Call failed: %v", err) + return err + } + + try.Status = fmt.Sprintf("Mined in %s (%d confirmations)", tx_result.Txs[0].MinedBlock[0], info.TopoHeight-bl_result.Block_Header.TopoHeight) + if try.Height + 2*config.BLOCK_BATCH_SIZE < (info.StableHeight + 1) { // successful confirmation w.account.PoolHistory = append(w.account.PoolHistory, w.account.Pool[i]) rlog.Infof("tx %s confirmed successfully at stableheight %d height %d trigger_height %d\n", try.TXID.String(), info.StableHeight, try.Height, w.account.Pool[i].Trigger_Height) w.account.Pool = append(w.account.Pool[:i], w.account.Pool[i+1:]...) i-- // so another element at same place gets used } continue - } else { - try.Status = fmt.Sprintf("Mined in sideblock (%d confirmations, waiting for more blocks)", info.Height-try.Height) - if try.Height < (info.StableHeight + 1) { // we have attempted, lets wait some blocks, this needs to be optimized, for instant transfer + } else { // tx may not be in pool and has not been mined, so we wait for couple of blocks + try.Status = fmt.Sprintf("unknown state (waiting for some more blocks)") + if try.Height + 2*config.BLOCK_BATCH_SIZE < (info.StableHeight + 1) { // we have attempted, lets wait some blocks, this needs to be optimized, for instant transfer // we need to send this tx again now }else{ continue // try other txs diff --git a/walletapi/wallet_transfer.go b/walletapi/wallet_transfer.go index 19a68b1..29e5183 100644 --- a/walletapi/wallet_transfer.go +++ b/walletapi/wallet_transfer.go @@ -23,7 +23,7 @@ import "fmt" //import cryptorand "crypto/rand" //import "encoding/binary" -import "encoding/hex" +//import "encoding/hex" //import "encoding/json" @@ -153,7 +153,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all var rings [][]*bn256.G1 var max_bits_array []int - _, self_e, _ := w.GetEncryptedBalanceAtTopoHeight(transfers[0].SCID, -1, w.GetAddress().String()) + _, self_e,height,topoheight,treehash, err := w.GetEncryptedBalanceAtTopoHeight(transfers[0].SCID, rpc.RECENT_BATCH_BLOCK, w.GetAddress().String()) if err != nil { fmt.Printf("self unregistered err %s\n", err) return @@ -162,17 +162,9 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all // WaitNewHeightBlock() // wait till a new block at new height is found // due to this we weill dispatch a new tx immediate after a block is found for better propagation - height := w.Daemon_Height - treehash := w.Merkle_Balance_TreeHash +// height := w.Daemon_Height +// treehash := w.Merkle_Balance_TreeHash - treehash_raw, err := hex.DecodeString(treehash) - if err != nil { - return - } - if len(treehash_raw) != 32 { - err = fmt.Errorf("roothash is not of 32 bytes, probably daemon corruption '%s'", treehash) - return - } for t := range transfers { @@ -187,7 +179,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all bits_needed := make([]int, ringsize, ringsize) - bits_needed[0], self_e, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, -1, w.GetAddress().String()) + bits_needed[0], self_e,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, int64(topoheight), w.GetAddress().String()) if err != nil { fmt.Printf("self unregistered err %s\n", err) return @@ -201,7 +193,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all return } var dest_e *crypto.ElGamal - bits_needed[1], dest_e, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, -1, addr.String()) + bits_needed[1], dest_e,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, int64(topoheight), addr.String()) if err != nil { fmt.Printf(" t %d unregistered1 '%s' %s\n", t, addr, err) return @@ -224,7 +216,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all // fmt.Printf("%s receiver %s sender %s\n", k, receiver_without_payment_id.String(), w.GetAddress().String()) var ebal *crypto.ElGamal var addr *rpc.Address - bits_needed[len(ring_members_keys)], ebal, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, -1, k) + bits_needed[len(ring_members_keys)], ebal,_,_,_, err = w.GetEncryptedBalanceAtTopoHeight(transfers[t].SCID, int64(topoheight), k) if err != nil { fmt.Printf(" unregistered %s\n", k) return @@ -262,7 +254,7 @@ func (w *Wallet_Memory) TransferPayload0(transfers []rpc.Transfer, transfer_all if !dry_run { rlog.Debugf("we should build a TX now") - tx = w.BuildTransaction(transfers, emap, rings, height, scdata, treehash_raw, max_bits_array) + tx = w.BuildTransaction(transfers, emap, rings, height, scdata, treehash[:], max_bits_array) } return