1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2013 The Bitcoin developers
3 // Distributed under the MIT/X11 software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
10 #include <boost/circular_buffer.hpp>
14 CTxMemPoolEntry::CTxMemPoolEntry():
15 nFee(0), nTxSize(0), nTime(0), dPriority(0.0)
17 nHeight = MEMPOOL_HEIGHT;
20 CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, int64_t _nFee,
21 int64_t _nTime, double _dPriority,
22 unsigned int _nHeight):
23 tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight)
25 nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
28 CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other)
34 CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const
36 int64_t nValueIn = tx.GetValueOut()+nFee;
37 double deltaPriority = ((double)(currentHeight-nHeight)*nValueIn)/nTxSize;
38 double dResult = dPriority + deltaPriority;
43 // Keep track of fee/priority for transactions confirmed within N blocks
48 boost::circular_buffer<CFeeRate> feeSamples;
49 boost::circular_buffer<double> prioritySamples;
51 template<typename T> std::vector<T> buf2vec(boost::circular_buffer<T> buf) const
53 std::vector<T> vec(buf.begin(), buf.end());
58 CBlockAverage() : feeSamples(100), prioritySamples(100) { }
60 void RecordFee(const CFeeRate& feeRate) {
61 feeSamples.push_back(feeRate);
64 void RecordPriority(double priority) {
65 prioritySamples.push_back(priority);
68 size_t FeeSamples() const { return feeSamples.size(); }
69 size_t GetFeeSamples(std::vector<CFeeRate>& insertInto) const
71 BOOST_FOREACH(const CFeeRate& f, feeSamples)
72 insertInto.push_back(f);
73 return feeSamples.size();
75 size_t PrioritySamples() const { return prioritySamples.size(); }
76 size_t GetPrioritySamples(std::vector<double>& insertInto) const
78 BOOST_FOREACH(double d, prioritySamples)
79 insertInto.push_back(d);
80 return prioritySamples.size();
83 // Used as belt-and-suspenders check when reading to detect
85 bool AreSane(const std::vector<CFeeRate>& vecFee, const CFeeRate& minRelayFee)
87 BOOST_FOREACH(CFeeRate fee, vecFee)
89 if (fee < CFeeRate(0))
91 if (fee.GetFeePerK() > minRelayFee.GetFeePerK() * 10000)
96 bool AreSane(const std::vector<double> vecPriority)
98 BOOST_FOREACH(double priority, vecPriority)
106 void Write(CAutoFile& fileout) const
108 std::vector<CFeeRate> vecFee = buf2vec(feeSamples);
110 std::vector<double> vecPriority = buf2vec(prioritySamples);
111 fileout << vecPriority;
114 void Read(CAutoFile& filein, const CFeeRate& minRelayFee) {
115 std::vector<CFeeRate> vecFee;
117 if (AreSane(vecFee, minRelayFee))
118 feeSamples.insert(feeSamples.end(), vecFee.begin(), vecFee.end());
120 throw runtime_error("Corrupt fee value in estimates file.");
121 std::vector<double> vecPriority;
122 filein >> vecPriority;
123 if (AreSane(vecPriority))
124 prioritySamples.insert(prioritySamples.end(), vecPriority.begin(), vecPriority.end());
126 throw runtime_error("Corrupt priority value in estimates file.");
127 if (feeSamples.size() + prioritySamples.size() > 0)
128 LogPrint("estimatefee", "Read %d fee samples and %d priority samples\n",
129 feeSamples.size(), prioritySamples.size());
133 class CMinerPolicyEstimator
136 // Records observed averages transactions that confirmed within one block, two blocks,
138 std::vector<CBlockAverage> history;
139 std::vector<CFeeRate> sortedFeeSamples;
140 std::vector<double> sortedPrioritySamples;
144 // nBlocksAgo is 0 based, i.e. transactions that confirmed in the highest seen block are
145 // nBlocksAgo == 0, transactions in the block before that are nBlocksAgo == 1 etc.
146 void seenTxConfirm(const CFeeRate& feeRate, const CFeeRate& minRelayFee, double dPriority, int nBlocksAgo)
148 // Last entry records "everything else".
149 int nBlocksTruncated = min(nBlocksAgo, (int) history.size() - 1);
150 assert(nBlocksTruncated >= 0);
152 // We need to guess why the transaction was included in a block-- either
153 // because it is high-priority or because it has sufficient fees.
154 bool sufficientFee = (feeRate > minRelayFee);
155 bool sufficientPriority = AllowFree(dPriority);
156 const char* assignedTo = "unassigned";
157 if (sufficientFee && !sufficientPriority)
159 history[nBlocksTruncated].RecordFee(feeRate);
162 else if (sufficientPriority && !sufficientFee)
164 history[nBlocksTruncated].RecordPriority(dPriority);
165 assignedTo = "priority";
169 // Neither or both fee and priority sufficient to get confirmed:
170 // don't know why they got confirmed.
172 LogPrint("estimatefee", "Seen TX confirm: %s : %s fee/%g priority, took %d blocks\n",
173 assignedTo, feeRate.ToString(), dPriority, nBlocksAgo);
177 CMinerPolicyEstimator(int nEntries) : nBestSeenHeight(0)
179 history.resize(nEntries);
182 void seenBlock(const std::vector<CTxMemPoolEntry>& entries, int nBlockHeight, const CFeeRate minRelayFee)
184 if (nBlockHeight <= nBestSeenHeight)
186 // Ignore side chains and re-orgs; assuming they are random
187 // they don't affect the estimate.
188 // And if an attacker can re-org the chain at will, then
189 // you've got much bigger problems than "attacker can influence
190 // transaction fees."
193 nBestSeenHeight = nBlockHeight;
195 // Fill up the history buckets based on how long transactions took
197 std::vector<std::vector<const CTxMemPoolEntry*> > entriesByConfirmations;
198 entriesByConfirmations.resize(history.size());
199 BOOST_FOREACH(const CTxMemPoolEntry& entry, entries)
201 // How many blocks did it take for miners to include this transaction?
202 int delta = nBlockHeight - entry.GetHeight();
205 // Re-org made us lose height, this should only happen if we happen
206 // to re-org on a difficulty transition point: very rare!
209 if ((delta-1) >= (int)history.size())
210 delta = history.size(); // Last bucket is catch-all
211 entriesByConfirmations.at(delta-1).push_back(&entry);
213 for (size_t i = 0; i < entriesByConfirmations.size(); i++)
215 std::vector<const CTxMemPoolEntry*> &e = entriesByConfirmations.at(i);
216 // Insert at most 10 random entries per bucket, otherwise a single block
217 // can dominate an estimate:
219 std::random_shuffle(e.begin(), e.end());
222 BOOST_FOREACH(const CTxMemPoolEntry* entry, e)
224 // Fees are stored and reported as BTC-per-kb:
225 CFeeRate feeRate(entry->GetFee(), entry->GetTxSize());
226 double dPriority = entry->GetPriority(entry->GetHeight()); // Want priority when it went IN
227 seenTxConfirm(feeRate, minRelayFee, dPriority, i);
231 //After new samples are added, we have to clear the sorted lists,
232 //so they'll be resorted the next time someone asks for an estimate
233 sortedFeeSamples.clear();
234 sortedPrioritySamples.clear();
236 for (size_t i = 0; i < history.size(); i++) {
237 if (history[i].FeeSamples() + history[i].PrioritySamples() > 0)
238 LogPrint("estimatefee", "estimates: for confirming within %d blocks based on %d/%d samples, fee=%s, prio=%g\n",
240 history[i].FeeSamples(), history[i].PrioritySamples(),
241 estimateFee(i+1).ToString(), estimatePriority(i+1));
245 // Can return CFeeRate(0) if we don't have any data for that many blocks back. nBlocksToConfirm is 1 based.
246 CFeeRate estimateFee(int nBlocksToConfirm)
250 if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size())
253 if (sortedFeeSamples.size() == 0)
255 for (size_t i = 0; i < history.size(); i++)
256 history.at(i).GetFeeSamples(sortedFeeSamples);
257 std::sort(sortedFeeSamples.begin(), sortedFeeSamples.end(),
258 std::greater<CFeeRate>());
260 if (sortedFeeSamples.size() < 11)
262 // Eleven is Gavin's Favorite Number
263 // ... but we also take a maximum of 10 samples per block so eleven means
264 // we're getting samples from at least two different blocks
268 int nBucketSize = history.at(nBlocksToConfirm).FeeSamples();
270 // Estimates should not increase as number of confirmations goes up,
271 // but the estimates are noisy because confirmations happen discretely
272 // in blocks. To smooth out the estimates, use all samples in the history
273 // and use the nth highest where n is (number of samples in previous bucket +
274 // half the samples in nBlocksToConfirm bucket):
275 size_t nPrevSize = 0;
276 for (int i = 0; i < nBlocksToConfirm; i++)
277 nPrevSize += history.at(i).FeeSamples();
278 size_t index = min(nPrevSize + nBucketSize/2, sortedFeeSamples.size()-1);
279 return sortedFeeSamples[index];
281 double estimatePriority(int nBlocksToConfirm)
285 if (nBlocksToConfirm < 0 || nBlocksToConfirm >= (int)history.size())
288 if (sortedPrioritySamples.size() == 0)
290 for (size_t i = 0; i < history.size(); i++)
291 history.at(i).GetPrioritySamples(sortedPrioritySamples);
292 std::sort(sortedPrioritySamples.begin(), sortedPrioritySamples.end(),
293 std::greater<double>());
295 if (sortedPrioritySamples.size() < 11)
298 int nBucketSize = history.at(nBlocksToConfirm).PrioritySamples();
300 // Estimates should not increase as number of confirmations needed goes up,
301 // but the estimates are noisy because confirmations happen discretely
302 // in blocks. To smooth out the estimates, use all samples in the history
303 // and use the nth highest where n is (number of samples in previous buckets +
304 // half the samples in nBlocksToConfirm bucket).
305 size_t nPrevSize = 0;
306 for (int i = 0; i < nBlocksToConfirm; i++)
307 nPrevSize += history.at(i).PrioritySamples();
308 size_t index = min(nPrevSize + nBucketSize/2, sortedPrioritySamples.size()-1);
309 return sortedPrioritySamples[index];
312 void Write(CAutoFile& fileout) const
314 fileout << nBestSeenHeight;
315 fileout << history.size();
316 BOOST_FOREACH(const CBlockAverage& entry, history)
318 entry.Write(fileout);
322 void Read(CAutoFile& filein, const CFeeRate& minRelayFee)
324 int nFileBestSeenHeight;
325 filein >> nFileBestSeenHeight;
327 filein >> numEntries;
328 if (numEntries <= 0 || numEntries > 10000)
329 throw runtime_error("Corrupt estimates file. Must have between 1 and 10k entires.");
331 std::vector<CBlockAverage> fileHistory;
333 for (size_t i = 0; i < numEntries; i++)
336 entry.Read(filein, minRelayFee);
337 fileHistory.push_back(entry);
340 //Now that we've processed the entire fee estimate data file and not
341 //thrown any errors, we can copy it to our history
342 nBestSeenHeight = nFileBestSeenHeight;
343 history = fileHistory;
344 assert(history.size() > 0);
349 CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) :
350 nTransactionsUpdated(0),
351 minRelayFee(_minRelayFee)
353 // Sanity checks off by default for performance, because otherwise
354 // accepting transactions becomes O(N^2) where N is the number
355 // of transactions in the pool
356 fSanityCheck = false;
358 // 25 blocks is a compromise between using a lot of disk/memory and
359 // trying to give accurate estimates to people who might be willing
360 // to wait a day or two to save a fraction of a penny in fees.
361 // Confirmation times for very-low-fee transactions that take more
362 // than an hour or three to confirm are highly variable.
363 minerPolicyEstimator = new CMinerPolicyEstimator(25);
366 CTxMemPool::~CTxMemPool()
368 delete minerPolicyEstimator;
371 void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins)
375 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.lower_bound(COutPoint(hashTx, 0));
377 // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx
378 while (it != mapNextTx.end() && it->first.hash == hashTx) {
379 coins.Spend(it->first.n); // and remove those outputs from coins
384 unsigned int CTxMemPool::GetTransactionsUpdated() const
387 return nTransactionsUpdated;
390 void CTxMemPool::AddTransactionsUpdated(unsigned int n)
393 nTransactionsUpdated += n;
397 bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry)
399 // Add to memory pool without checking anything.
400 // Used by main.cpp AcceptToMemoryPool(), which DOES do
401 // all the appropriate checks.
405 const CTransaction& tx = mapTx[hash].GetTx();
406 for (unsigned int i = 0; i < tx.vin.size(); i++)
407 mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i);
408 nTransactionsUpdated++;
409 totalTxSize += entry.GetTxSize();
415 void CTxMemPool::remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive)
417 // Remove transaction from memory pool
420 uint256 hash = tx.GetHash();
422 for (unsigned int i = 0; i < tx.vout.size(); i++) {
423 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(COutPoint(hash, i));
424 if (it == mapNextTx.end())
426 remove(*it->second.ptx, removed, true);
429 if (mapTx.count(hash))
431 removed.push_front(tx);
432 BOOST_FOREACH(const CTxIn& txin, tx.vin)
433 mapNextTx.erase(txin.prevout);
435 totalTxSize -= mapTx[hash].GetTxSize();
437 nTransactionsUpdated++;
442 void CTxMemPool::removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed)
444 // Remove transactions which depend on inputs of tx, recursively
445 list<CTransaction> result;
447 BOOST_FOREACH(const CTxIn &txin, tx.vin) {
448 std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(txin.prevout);
449 if (it != mapNextTx.end()) {
450 const CTransaction &txConflict = *it->second.ptx;
451 if (txConflict != tx)
453 remove(txConflict, removed, true);
459 // Called when a block is connected. Removes from mempool and updates the miner fee estimator.
460 void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned int nBlockHeight,
461 std::list<CTransaction>& conflicts)
464 std::vector<CTxMemPoolEntry> entries;
465 BOOST_FOREACH(const CTransaction& tx, vtx)
467 uint256 hash = tx.GetHash();
468 if (mapTx.count(hash))
469 entries.push_back(mapTx[hash]);
471 minerPolicyEstimator->seenBlock(entries, nBlockHeight, minRelayFee);
472 BOOST_FOREACH(const CTransaction& tx, vtx)
474 std::list<CTransaction> dummy;
475 remove(tx, dummy, false);
476 removeConflicts(tx, conflicts);
477 ClearPrioritisation(tx.GetHash());
482 void CTxMemPool::clear()
488 ++nTransactionsUpdated;
491 void CTxMemPool::check(const CCoinsViewCache *pcoins) const
496 LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size());
498 uint64_t checkTotal = 0;
501 for (std::map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) {
503 checkTotal += it->second.GetTxSize();
504 const CTransaction& tx = it->second.GetTx();
505 BOOST_FOREACH(const CTxIn &txin, tx.vin) {
506 // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's.
507 std::map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(txin.prevout.hash);
508 if (it2 != mapTx.end()) {
509 const CTransaction& tx2 = it2->second.GetTx();
510 assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull());
512 const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash);
513 assert(coins && coins->IsAvailable(txin.prevout.n));
515 // Check whether its inputs are marked in mapNextTx.
516 std::map<COutPoint, CInPoint>::const_iterator it3 = mapNextTx.find(txin.prevout);
517 assert(it3 != mapNextTx.end());
518 assert(it3->second.ptx == &tx);
519 assert(it3->second.n == i);
523 for (std::map<COutPoint, CInPoint>::const_iterator it = mapNextTx.begin(); it != mapNextTx.end(); it++) {
524 uint256 hash = it->second.ptx->GetHash();
525 map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(hash);
526 const CTransaction& tx = it2->second.GetTx();
527 assert(it2 != mapTx.end());
528 assert(&tx == it->second.ptx);
529 assert(tx.vin.size() > it->second.n);
530 assert(it->first == it->second.ptx->vin[it->second.n].prevout);
533 assert(totalTxSize == checkTotal);
536 void CTxMemPool::queryHashes(vector<uint256>& vtxid)
541 vtxid.reserve(mapTx.size());
542 for (map<uint256, CTxMemPoolEntry>::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi)
543 vtxid.push_back((*mi).first);
546 bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const
549 map<uint256, CTxMemPoolEntry>::const_iterator i = mapTx.find(hash);
550 if (i == mapTx.end()) return false;
551 result = i->second.GetTx();
555 CFeeRate CTxMemPool::estimateFee(int nBlocks) const
558 return minerPolicyEstimator->estimateFee(nBlocks);
560 double CTxMemPool::estimatePriority(int nBlocks) const
563 return minerPolicyEstimator->estimatePriority(nBlocks);
567 CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const
571 fileout << 99900; // version required to read: 0.9.99 or later
572 fileout << CLIENT_VERSION; // version that wrote the file
573 minerPolicyEstimator->Write(fileout);
575 catch (std::exception &e) {
576 LogPrintf("CTxMemPool::WriteFeeEstimates() : unable to write policy estimator data (non-fatal)");
583 CTxMemPool::ReadFeeEstimates(CAutoFile& filein)
586 int nVersionRequired, nVersionThatWrote;
587 filein >> nVersionRequired >> nVersionThatWrote;
588 if (nVersionRequired > CLIENT_VERSION)
589 return error("CTxMemPool::ReadFeeEstimates() : up-version (%d) fee estimate file", nVersionRequired);
592 minerPolicyEstimator->Read(filein, minRelayFee);
594 catch (std::exception &e) {
595 LogPrintf("CTxMemPool::ReadFeeEstimates() : unable to read policy estimator data (non-fatal)");
601 void CTxMemPool::PrioritiseTransaction(const uint256 hash, const string strHash, double dPriorityDelta, int64_t nFeeDelta)
605 std::pair<double, int64_t> &deltas = mapDeltas[hash];
606 deltas.first += dPriorityDelta;
607 deltas.second += nFeeDelta;
609 LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, nFeeDelta);
612 void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, int64_t &nFeeDelta)
615 std::map<uint256, std::pair<double, int64_t> >::iterator pos = mapDeltas.find(hash);
616 if (pos == mapDeltas.end())
618 const std::pair<double, int64_t> &deltas = pos->second;
619 dPriorityDelta += deltas.first;
620 nFeeDelta += deltas.second;
623 void CTxMemPool::ClearPrioritisation(const uint256 hash)
626 mapDeltas.erase(hash);
630 CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { }
632 bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const {
633 // If an entry in the mempool exists, always return that one, as it's guaranteed to never
634 // conflict with the underlying cache, and it cannot have pruned entries (as it contains full)
635 // transactions. First checking the underlying cache risks returning a pruned entry instead.
637 if (mempool.lookup(txid, tx)) {
638 coins = CCoins(tx, MEMPOOL_HEIGHT);
641 return (base->GetCoins(txid, coins) && !coins.IsPruned());
644 bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const {
645 return mempool.exists(txid) || base->HaveCoins(txid);