1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2013 The Bitcoin developers
3 // Distributed under the MIT/X11 software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
9 #include <boost/circular_buffer.hpp>
13 CTxMemPoolEntry::CTxMemPoolEntry()
15 nHeight = MEMPOOL_HEIGHT;
18 CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, int64_t _nFee,
19 int64_t _nTime, double _dPriority,
20 unsigned int _nHeight):
21 tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight)
23 nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
26 CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other)
32 CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const
34 int64_t nValueIn = tx.GetValueOut()+nFee;
35 double deltaPriority = ((double)(currentHeight-nHeight)*nValueIn)/nTxSize;
36 double dResult = dPriority + deltaPriority;
41 // Keep track of fee/priority for transactions confirmed within N blocks
46 boost::circular_buffer<CFeeRate> feeSamples;
47 boost::circular_buffer<double> prioritySamples;
49 template<typename T> std::vector<T> buf2vec(boost::circular_buffer<T> buf) const
51 std::vector<T> vec(buf.begin(), buf.end());
56 CBlockAverage() : feeSamples(100), prioritySamples(100) { }
58 void RecordFee(const CFeeRate& feeRate) {
59 feeSamples.push_back(feeRate);
62 void RecordPriority(double priority) {
63 prioritySamples.push_back(priority);
66 size_t FeeSamples() const { return feeSamples.size(); }
67 size_t GetFeeSamples(std::vector<CFeeRate>& insertInto) const
69 BOOST_FOREACH(const CFeeRate& f, feeSamples)
70 insertInto.push_back(f);
71 return feeSamples.size();
73 size_t PrioritySamples() const { return prioritySamples.size(); }
74 size_t GetPrioritySamples(std::vector<double>& insertInto) const
76 BOOST_FOREACH(double d, prioritySamples)
77 insertInto.push_back(d);
78 return prioritySamples.size();
81 // Used as belt-and-suspenders check when reading to detect
83 bool AreSane(const std::vector<CFeeRate>& vecFee, const CFeeRate& minRelayFee)
85 BOOST_FOREACH(CFeeRate fee, vecFee)
87 if (fee < CFeeRate(0))
89 if (fee.GetFeePerK() > minRelayFee.GetFeePerK() * 10000)
94 bool AreSane(const std::vector<double> vecPriority)
96 BOOST_FOREACH(double priority, vecPriority)
104 void Write(CAutoFile& fileout) const
106 std::vector<CFeeRate> vecFee = buf2vec(feeSamples);
108 std::vector<double> vecPriority = buf2vec(prioritySamples);
109 fileout << vecPriority;
112 void Read(CAutoFile& filein, const CFeeRate& minRelayFee) {
113 std::vector<CFeeRate> vecFee;
115 if (AreSane(vecFee, minRelayFee))
116 feeSamples.insert(feeSamples.end(), vecFee.begin(), vecFee.end());
118 throw runtime_error("Corrupt fee value in estimates file.");
119 std::vector<double> vecPriority;
120 filein >> vecPriority;
121 if (AreSane(vecPriority))
122 prioritySamples.insert(prioritySamples.end(), vecPriority.begin(), vecPriority.end());
124 throw runtime_error("Corrupt priority value in estimates file.");
125 if (feeSamples.size() + prioritySamples.size() > 0)
126 LogPrint("estimatefee", "Read %d fee samples and %d priority samples\n",
127 feeSamples.size(), prioritySamples.size());
131 class CMinerPolicyEstimator
134 // Records observed averages transactions that confirmed within one block, two blocks,
136 std::vector<CBlockAverage> history;
137 std::vector<CFeeRate> sortedFeeSamples;
138 std::vector<double> sortedPrioritySamples;
142 // nBlocksAgo is 0 based, i.e. transactions that confirmed in the highest seen block are
143 // nBlocksAgo == 0, transactions in the block before that are nBlocksAgo == 1 etc.
144 void seenTxConfirm(const CFeeRate& feeRate, const CFeeRate& minRelayFee, double dPriority, int nBlocksAgo)
146 // Last entry records "everything else".
147 int nBlocksTruncated = min(nBlocksAgo, (int) history.size() - 1);
148 assert(nBlocksTruncated >= 0);
150 // We need to guess why the transaction was included in a block-- either
151 // because it is high-priority or because it has sufficient fees.
152 bool sufficientFee = (feeRate > minRelayFee);
153 bool sufficientPriority = AllowFree(dPriority);
154 const char* assignedTo = "unassigned";
155 if (sufficientFee && !sufficientPriority)
157 history[nBlocksTruncated].RecordFee(feeRate);
160 else if (sufficientPriority && !sufficientFee)
162 history[nBlocksTruncated].RecordPriority(dPriority);
163 assignedTo = "priority";
167 // Neither or both fee and priority sufficient to get confirmed:
168 // don't know why they got confirmed.
170 LogPrint("estimatefee", "Seen TX confirm: %s : %s fee/%g priority, took %d blocks\n",
171 assignedTo, feeRate.ToString(), dPriority, nBlocksAgo);
175 CMinerPolicyEstimator(int nEntries) : nBestSeenHeight(0)
177 history.resize(nEntries);
180 void seenBlock(const std::vector<CTxMemPoolEntry>& entries, int nBlockHeight, const CFeeRate minRelayFee)
182 if (nBlockHeight <= nBestSeenHeight)
184 // Ignore side chains and re-orgs; assuming they are random
185 // they don't affect the estimate.
186 // And if an attacker can re-org the chain at will, then
187 // you've got much bigger problems than "attacker can influence
188 // transaction fees."
191 nBestSeenHeight = nBlockHeight;
193 // Fill up the history buckets based on how long transactions took
195 std::vector<std::vector<const CTxMemPoolEntry*> > entriesByConfirmations;
196 entriesByConfirmations.resize(history.size());
197 BOOST_FOREACH(const CTxMemPoolEntry& entry, entries)
199 // How many blocks did it take for miners to include this transaction?
200 int delta = nBlockHeight - entry.GetHeight();
203 // Re-org made us lose height, this should only happen if we happen
204 // to re-org on a difficulty transition point: very rare!
207 if ((delta-1) >= (int)history.size())
208 delta = history.size(); // Last bucket is catch-all
209 entriesByConfirmations[delta-1].push_back(&entry);
211 for (size_t i = 0; i < entriesByConfirmations.size(); i++)
213 std::vector<const CTxMemPoolEntry*> &e = entriesByConfirmations.at(i);
214 // Insert at most 10 random entries per bucket, otherwise a single block
215 // can dominate an estimate:
217 std::random_shuffle(e.begin(), e.end());
220 BOOST_FOREACH(const CTxMemPoolEntry* entry, e)
222 // Fees are stored and reported as BTC-per-kb:
223 CFeeRate feeRate(entry->GetFee(), entry->GetTxSize());
224 double dPriority = entry->GetPriority(entry->GetHeight()); // Want priority when it went IN
225 seenTxConfirm(feeRate, minRelayFee, dPriority, i);
228 for (size_t i = 0; i < history.size(); i++) {
229 if (history[i].FeeSamples() + history[i].PrioritySamples() > 0)
230 LogPrint("estimatefee", "estimates: for confirming within %d blocks based on %d/%d samples, fee=%s, prio=%g\n",
232 history[i].FeeSamples(), history[i].PrioritySamples(),
233 estimateFee(i+1).ToString(), estimatePriority(i+1));
235 sortedFeeSamples.clear();
236 sortedPrioritySamples.clear();
239 // Can return CFeeRate(0) if we don't have any data for that many blocks back. nBlocksToConfirm is 1 based.
240 CFeeRate estimateFee(int nBlocksToConfirm)
244 if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size())
247 if (sortedFeeSamples.size() == 0)
249 for (size_t i = 0; i < history.size(); i++)
250 history.at(i).GetFeeSamples(sortedFeeSamples);
251 std::sort(sortedFeeSamples.begin(), sortedFeeSamples.end(),
252 std::greater<CFeeRate>());
254 if (sortedFeeSamples.size() < 11)
256 // Eleven is Gavin's Favorite Number
257 // ... but we also take a maximum of 10 samples per block so eleven means
258 // we're getting samples from at least two different blocks
262 int nBucketSize = history.at(nBlocksToConfirm).FeeSamples();
264 // Estimates should not increase as number of confirmations goes up,
265 // but the estimates are noisy because confirmations happen discretely
266 // in blocks. To smooth out the estimates, use all samples in the history
267 // and use the nth highest where n is (number of samples in previous bucket +
268 // half the samples in nBlocksToConfirm bucket):
269 size_t nPrevSize = 0;
270 for (int i = 0; i < nBlocksToConfirm; i++)
271 nPrevSize += history.at(i).FeeSamples();
272 size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1);
273 return sortedFeeSamples[index];
275 double estimatePriority(int nBlocksToConfirm)
279 if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size())
282 if (sortedPrioritySamples.size() == 0)
284 for (size_t i = 0; i < history.size(); i++)
285 history.at(i).GetPrioritySamples(sortedPrioritySamples);
286 std::sort(sortedPrioritySamples.begin(), sortedPrioritySamples.end(),
287 std::greater<double>());
289 if (sortedPrioritySamples.size() < 11)
292 int nBucketSize = history.at(nBlocksToConfirm).PrioritySamples();
294 // Estimates should not increase as number of confirmations needed goes up,
295 // but the estimates are noisy because confirmations happen discretely
296 // in blocks. To smooth out the estimates, use all samples in the history
297 // and use the nth highest where n is (number of samples in previous buckets +
298 // half the samples in nBlocksToConfirm bucket).
299 size_t nPrevSize = 0;
300 for (int i = 0; i < nBlocksToConfirm; i++)
301 nPrevSize += history.at(i).PrioritySamples();
302 size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1);
303 return sortedPrioritySamples[index];
306 void Write(CAutoFile& fileout) const
308 fileout << nBestSeenHeight;
309 fileout << history.size();
310 BOOST_FOREACH(const CBlockAverage& entry, history)
312 entry.Write(fileout);
316 void Read(CAutoFile& filein, const CFeeRate& minRelayFee)
318 filein >> nBestSeenHeight;
320 filein >> numEntries;
322 for (size_t i = 0; i < numEntries; i++)
325 entry.Read(filein, minRelayFee);
326 history.push_back(entry);
332 CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : minRelayFee(_minRelayFee)
334 // Sanity checks off by default for performance, because otherwise
335 // accepting transactions becomes O(N^2) where N is the number
336 // of transactions in the pool
337 fSanityCheck = false;
339 // 25 blocks is a compromise between using a lot of disk/memory and
340 // trying to give accurate estimates to people who might be willing
341 // to wait a day or two to save a fraction of a penny in fees.
342 // Confirmation times for very-low-fee transactions that take more
343 // than an hour or three to confirm are highly variable.
344 minerPolicyEstimator = new CMinerPolicyEstimator(25);
347 CTxMemPool::~CTxMemPool()
349 delete minerPolicyEstimator;
352 void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins)
356 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.lower_bound(COutPoint(hashTx, 0));
358 // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx
359 while (it != mapNextTx.end() && it->first.hash == hashTx) {
360 coins.Spend(it->first.n); // and remove those outputs from coins
365 unsigned int CTxMemPool::GetTransactionsUpdated() const
368 return nTransactionsUpdated;
371 void CTxMemPool::AddTransactionsUpdated(unsigned int n)
374 nTransactionsUpdated += n;
378 bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry)
380 // Add to memory pool without checking anything.
381 // Used by main.cpp AcceptToMemoryPool(), which DOES do
382 // all the appropriate checks.
386 const CTransaction& tx = mapTx[hash].GetTx();
387 for (unsigned int i = 0; i < tx.vin.size(); i++)
388 mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i);
389 nTransactionsUpdated++;
395 void CTxMemPool::remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive)
397 // Remove transaction from memory pool
400 uint256 hash = tx.GetHash();
402 for (unsigned int i = 0; i < tx.vout.size(); i++) {
403 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(COutPoint(hash, i));
404 if (it == mapNextTx.end())
406 remove(*it->second.ptx, removed, true);
409 if (mapTx.count(hash))
411 removed.push_front(tx);
412 BOOST_FOREACH(const CTxIn& txin, tx.vin)
413 mapNextTx.erase(txin.prevout);
415 nTransactionsUpdated++;
420 void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed)
422 // Remove transactions which depend on inputs of tx, recursively
423 list<CTransaction> result;
425 BOOST_FOREACH(const CTxIn &txin, tx.vin) {
426 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(txin.prevout);
427 if (it != mapNextTx.end()) {
428 const CTransaction &txConflict = *it->second.ptx;
429 if (txConflict != tx)
431 remove(txConflict, removed, true);
437 // Called when a block is connected. Removes from mempool and updates the miner fee estimator.
438 void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight,
439 std::list<CTransaction>& conflicts)
442 std::vector<CTxMemPoolEntry> entries;
443 BOOST_FOREACH(const CTransaction& tx, vtx)
445 uint256 hash = tx.GetHash();
446 if (mapTx.count(hash))
447 entries.push_back(mapTx[hash]);
449 minerPolicyEstimator->seenBlock(entries, nBlockHeight, minRelayFee);
450 BOOST_FOREACH(const CTransaction& tx, vtx)
452 std::list<CTransaction> dummy;
453 remove(tx, dummy, false);
454 removeConflicts(tx, conflicts);
455 ClearPrioritisation(tx.GetHash());
460 void CTxMemPool::clear()
465 ++nTransactionsUpdated;
468 void CTxMemPool::check(CCoinsViewCache *pcoins) const
473 LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size());
476 for (std::map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) {
478 const CTransaction& tx = it->second.GetTx();
479 BOOST_FOREACH(const CTxIn &txin, tx.vin) {
480 // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's.
481 std::map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(txin.prevout.hash);
482 if (it2 != mapTx.end()) {
483 const CTransaction& tx2 = it2->second.GetTx();
484 assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull());
486 CCoins &coins = pcoins->GetCoins(txin.prevout.hash);
487 assert(coins.IsAvailable(txin.prevout.n));
489 // Check whether its inputs are marked in mapNextTx.
490 std::map<COutPoint, CInPoint>::const_iterator it3 = mapNextTx.find(txin.prevout);
491 assert(it3 != mapNextTx.end());
492 assert(it3->second.ptx == &tx);
493 assert(it3->second.n == i);
497 for (std::map<COutPoint, CInPoint>::const_iterator it = mapNextTx.begin(); it != mapNextTx.end(); it++) {
498 uint256 hash = it->second.ptx->GetHash();
499 map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(hash);
500 const CTransaction& tx = it2->second.GetTx();
501 assert(it2 != mapTx.end());
502 assert(&tx == it->second.ptx);
503 assert(tx.vin.size() > it->second.n);
504 assert(it->first == it->second.ptx->vin[it->second.n].prevout);
508 void CTxMemPool::queryHashes(vector<uint256>& vtxid)
513 vtxid.reserve(mapTx.size());
514 for (map<uint256, CTxMemPoolEntry>::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi)
515 vtxid.push_back((*mi).first);
518 bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const
521 map<uint256, CTxMemPoolEntry>::const_iterator i = mapTx.find(hash);
522 if (i == mapTx.end()) return false;
523 result = i->second.GetTx();
527 CFeeRate CTxMemPool::estimateFee(int nBlocks) const
530 return minerPolicyEstimator->estimateFee(nBlocks);
532 double CTxMemPool::estimatePriority(int nBlocks) const
535 return minerPolicyEstimator->estimatePriority(nBlocks);
539 CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const
543 fileout << 99900; // version required to read: 0.9.99 or later
544 fileout << CLIENT_VERSION; // version that wrote the file
545 minerPolicyEstimator->Write(fileout);
547 catch (std::exception &e) {
548 LogPrintf("CTxMemPool::WriteFeeEstimates() : unable to write policy estimator data (non-fatal)");
555 CTxMemPool::ReadFeeEstimates(CAutoFile& filein)
558 int nVersionRequired, nVersionThatWrote;
559 filein >> nVersionRequired >> nVersionThatWrote;
560 if (nVersionRequired > CLIENT_VERSION)
561 return error("CTxMemPool::ReadFeeEstimates() : up-version (%d) fee estimate file", nVersionRequired);
564 minerPolicyEstimator->Read(filein, minRelayFee);
566 catch (std::exception &e) {
567 LogPrintf("CTxMemPool::ReadFeeEstimates() : unable to read policy estimator data (non-fatal)");
573 void CTxMemPool::PrioritiseTransaction(const uint256 hash, const string strHash, double dPriorityDelta, int64_t nFeeDelta)
577 std::pair<double, int64_t> &deltas = mapDeltas[hash];
578 deltas.first += dPriorityDelta;
579 deltas.second += nFeeDelta;
581 LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash.c_str(), dPriorityDelta, nFeeDelta);
584 void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, int64_t &nFeeDelta)
587 std::map<uint256, std::pair<double, int64_t> >::iterator pos = mapDeltas.find(hash);
588 if (pos == mapDeltas.end())
590 const std::pair<double, int64_t> &deltas = pos->second;
591 dPriorityDelta += deltas.first;
592 nFeeDelta += deltas.second;
595 void CTxMemPool::ClearPrioritisation(const uint256 hash)
598 mapDeltas.erase(hash);
602 CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { }
604 bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) {
605 // If an entry in the mempool exists, always return that one, as it's guaranteed to never
606 // conflict with the underlying cache, and it cannot have pruned entries (as it contains full)
607 // transactions. First checking the underlying cache risks returning a pruned entry instead.
609 if (mempool.lookup(txid, tx)) {
610 coins = CCoins(tx, MEMPOOL_HEIGHT);
613 return (base->GetCoins(txid, coins) && !coins.IsPruned());
616 bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) {
617 return mempool.exists(txid) || base->HaveCoins(txid);