Merge pull request #2114 from matiu/ref/utxos-cache

Ref/utxos cache

js/controllers/copayers.js

@@ -2,9 +2,6 @@
 
 angular.module('copayApp.controllers').controller('CopayersController',
   function($scope, $rootScope, $timeout, go) {
-
-    console.log('[copayers.js.5]'); //TODO
-
     $scope.init = function() {
       var w = $rootScope.wallet;
       $rootScope.title = 'Waiting copayers for ' + $rootScope.wallet.getName();

js/models/Wallet.js

@@ -98,7 +98,6 @@ function Wallet(opts) {
   this.syncedTimestamp = opts.syncedTimestamp || 0;
   this.lastMessageFrom = {};
 
-  this.paymentRequestsCache = {};
 
   var networkName = Wallet.obtainNetworkName(opts);
 
@@ -113,6 +112,10 @@ function Wallet(opts) {
   //one nonce for oneself, and then one nonce for each copayer
   this.network.setHexNonce(opts.networkNonce);
   this.network.setHexNonces(opts.networkNonces);
+
+  this.cache = {
+    paymentRequests: {},
+  };
 }
 
 inherits(Wallet, events.EventEmitter);
@@ -255,29 +258,35 @@ Wallet.prototype.seedCopayer = function(pubKey) {
 };
 
 
-Wallet.prototype._newAddresses = function(dontUpdateUx) {
-  this.subscribeToAddresses();
-  this.emitAndKeepAlive('newAddresses', dontUpdateUx);
-};
-
-
 /**
  * @desc Handles an 'indexes' message.
  *
  * Processes the data using {@link HDParams#fromList} and merges it with the
  * {@link Wallet#publicKeyRing}.
  *
- * @param {string} senderId - the sender id
  * @param {Object} data - the data recived, {@see HDParams#fromList}
  */
-Wallet.prototype._onIndexes = function(senderId, data) {
+Wallet.prototype._doOnIndexes = function(indexes, fromTxProposal) {
-  var inIndexes = HDParams.fromList(data.indexes);
+  preconditions.checkArgument(indexes);
+  var inIndexes = HDParams.fromList(indexes);
   var hasChanged = this.publicKeyRing.mergeIndexes(inIndexes);
   if (hasChanged) {
-    this._newAddresses();
+
+    // If the new indexes come from TX proposals (change address)
+    // that addresses should be new. No need to clean cache.
+    if (!fromTxProposal)
+      this.clearUnspentCache();
+
+    this.subscribeToAddresses();
+    this.emitAndKeepAlive('newAddresses');
   }
 };
 
+
+Wallet.prototype._onIndexes = function(senderId, data) {
+  return this._doOnIndexes(data.indexes);
+};
+
 /**
  * @desc
  * Changes wallet settings. The settings format is:
@@ -517,6 +526,10 @@ Wallet.prototype._onTxProposal = function(senderId, data) {
   preconditions.checkArgument(data.txProposal);
   var self = this;
 
+  if (data.indexes) {
+    this._doOnIndexes(data.indexes, true);
+  }
+
   try {
     var incomingTx = self._txProposalFromUntrustedObj(data.txProposal, Wallet.builderOpts);
     var incomingNtxid = incomingTx.getId();
@@ -726,7 +739,7 @@ Wallet.prototype._onData = function(senderId, data, ts) {
       this._onSignature(senderId, data);
       break;
     case 'indexes':
-      this._onIndexes(senderId, data);
+      this._onIndexes(data.indexes);
       break;
     case 'addressbook':
       this._onAddressBook(senderId, data);
@@ -886,6 +899,7 @@ Wallet.prototype._setupBlockchainHandlers = function() {
   log.debug('Setting Blockchain listeners for', this.getName());
   self.blockchain.on('reconnect', function(attempts) {
     log.debug('Wallet:' + self.id + 'blockchain reconnect event');
+    self.clearUnspentCache();
     self.emitAndKeepAlive('insightReconnected');
   });
 
@@ -897,12 +911,19 @@ Wallet.prototype._setupBlockchainHandlers = function() {
   self.blockchain.on('tx', function(tx) {
     log.debug('Wallet:' + self.id + ' blockchain tx event');
     var addresses = self.getAddresses();
+    // This should always be >=0
     if (_.indexOf(addresses, tx.address) >= 0) {
+      self.clearUnspentCache();
       self.emitAndKeepAlive('tx', tx.address, self.addressIsChange(tx.address));
     }
   });
 
   if (!self.spendUnconfirmed) {
+
+    // TODO HERE should only clean utxos if there are some wallet
+    // transactions waiting for confirmation (ie confirmation < min confirmation)
+    self.clearUnspentCache();
+
     self.blockchain.on('block', self.emitAndKeepAlive.bind(self, 'balanceUpdated'));
   }
 }
@@ -1204,10 +1225,12 @@ Wallet.prototype.sendAllTxProposals = function(recipients, sinceTs) {
  */
 Wallet.prototype.sendTxProposal = function(ntxid, recipients) {
   preconditions.checkArgument(ntxid);
+  var indexes = HDParams.serialize(this.publicKeyRing.indexes);
   this._sendToPeers(recipients, {
     type: 'txProposal',
     txProposal: this.txProposals.get(ntxid).toObjTrim(),
     walletId: this.id,
+    indexes: indexes,
   });
 };
 
@@ -1355,7 +1378,10 @@ Wallet.prototype.getName = function() {
  * @return {string[]} a list of all the addresses generated so far for the wallet
  */
 Wallet.prototype._doGenerateAddress = function(isChange) {
-  return this.publicKeyRing.generateAddress(isChange, this.publicKey);
+  var addr = this.publicKeyRing.generateAddress(isChange, this.publicKey);
+  this.subscribeToAddresses();
+  this.emitAndKeepAlive('newAddresses');
+  return addr;
 };
 
 /**
@@ -1366,7 +1392,6 @@ Wallet.prototype._doGenerateAddress = function(isChange) {
 Wallet.prototype.generateAddress = function(isChange) {
   var addr = this._doGenerateAddress(isChange);
   this.sendIndexes();
-  this._newAddresses();
   return addr;
 };
 
@@ -1666,8 +1691,8 @@ Wallet.prototype.fetchPaymentRequest = function(options, cb) {
   preconditions.checkArgument(options.url.indexOf('http') == 0, 'Bad PayPro URL given:' + options.url);
   var self = this;
 
-  if (self.paymentRequestsCache[options.url])
+  if (self.cache.paymentRequests[options.url])
-    return cb(null, self.paymentRequestsCache[options.url]);
+    return cb(null, self.cache.paymentRequests[options.url]);
 
   this.httpUtil.request({
     method: 'GET',
@@ -1689,7 +1714,7 @@ Wallet.prototype.fetchPaymentRequest = function(options, cb) {
 
       log.debug('PayPro request data', merchantData);
 
-      self.paymentRequestsCache[options.url] = merchantData;
+      self.cache.paymentRequests[options.url] = merchantData;
       return cb(err, merchantData);
     })
     .error(function(data, status) {
@@ -2076,6 +2101,7 @@ Wallet.estimatedFee = function(unspentCount) {
   return parseInt(estimatedSizeKb * bitcore.TransactionBuilder.FEE_PER_1000B_SAT);
 };
 
+
 /**
  * @callback {getBalanceCallback}
  * @param {string=} err - an error, if any
@@ -2084,45 +2110,58 @@ Wallet.estimatedFee = function(unspentCount) {
  * @param {number} safeBalance - total number of satoshis in UTXOs that are not part of any TxProposal
  * @param {number} safeUnspentCount - total number of safe unspent Outputs that make this balance.
  */
+
 /**
- * @desc Returns the balances for all addresses in Satoshis
+ * computeBalance
+ *
+ * @param safeUnspent
+ * @param unspent
  * @param {getBalanceCallback} cb
  */
-Wallet.prototype.getBalance = function(cb) {
+Wallet.prototype.computeBalance = function(safeUnspent, unspent, cb) {
   var balance = 0;
   var safeBalance = 0;
   var balanceByAddr = {};
   var COIN = coinUtil.COIN;
 
+  for (var i = 0; i < unspent.length; i++) {
+    var u = unspent[i];
+    var amt = u.amount * COIN;
+    balance += amt;
+    balanceByAddr[u.address] = (balanceByAddr[u.address] || 0) + amt;
+  }
+
+  // we multiply and divide by BIT to avoid rounding errors when adding
+  for (var a in balanceByAddr) {
+    balanceByAddr[a] = parseInt(balanceByAddr[a].toFixed(0), 10);
+  }
+
+  balance = parseInt(balance.toFixed(0), 10);
+
+  var safeUnspentCount = safeUnspent.length;
+
+  for (var i = 0; i < safeUnspentCount; i++) {
+    var u = safeUnspent[i];
+    var amt = u.amount * COIN;
+    safeBalance += amt;
+  }
+
+  safeBalance = parseInt(safeBalance.toFixed(0), 10);
+  return cb(null, balance, balanceByAddr, safeBalance, safeUnspentCount);
+};
+
+/**
+ * @desc Returns the balances for all addresses in Satoshis
+ * @param {getBalanceCallback} cb
+ */
+Wallet.prototype.getBalance = function(cb) {
+  var self = this;
+
   this.getUnspent(function(err, safeUnspent, unspent) {
     if (err) {
       return cb(err);
     }
-
+    self.computeBalance(safeUnspent, unspent, cb);
-    for (var i = 0; i < unspent.length; i++) {
-      var u = unspent[i];
-      var amt = u.amount * COIN;
-      balance += amt;
-      balanceByAddr[u.address] = (balanceByAddr[u.address] || 0) + amt;
-    }
-
-    // we multiply and divide by BIT to avoid rounding errors when adding
-    for (var a in balanceByAddr) {
-      balanceByAddr[a] = parseInt(balanceByAddr[a].toFixed(0), 10);
-    }
-
-    balance = parseInt(balance.toFixed(0), 10);
-
-    var safeUnspentCount = safeUnspent.length;
-
-    for (var i = 0; i < safeUnspentCount; i++) {
-      var u = safeUnspent[i];
-      var amt = u.amount * COIN;
-      safeBalance += amt;
-    }
-
-    safeBalance = parseInt(safeBalance.toFixed(0), 10);
-    return cb(null, balance, balanceByAddr, safeBalance, safeUnspentCount);
   });
 };
 
@@ -2140,6 +2179,12 @@ Wallet.prototype.maxRejectCount = function() {
   return this.totalCopayers - this.requiredCopayers;
 };
 
+
+Wallet.prototype.clearUnspentCache = function() {
+  log.debug('Cleaning unspent cache');
+  this.cache.unspent = null;
+};
+
 /**
  * @callback getUnspentCallback
  * @desc Get a list of unspent transaction outputs
@@ -2149,33 +2194,55 @@ Wallet.prototype.maxRejectCount = function() {
  * @param {getUnspentCallback} cb
  */
 
-// TODO: Can we add cache to getUnspent?
 Wallet.prototype.getUnspent = function(cb) {
   var self = this;
+
+
+  if (self.cache.unspent != null) {
+    log.debug('Wallet ' + this.getName() + ': Get unspent cache hit');
+    return self.computeUnspent(self.cache.unspent, cb);
+    return 
+  }
+
   var addresses = this.getAddresses();
-
-
   log.debug('Wallet ' + this.getName() + ': Getting unspents from ' + addresses.length + ' addresses');
   this.blockchain.getUnspent(addresses, function(err, unspentList) {
-
+    if (err)
-    if (err) {
       return cb(err);
-    }
 
-    var safeUnspendList = [];
+    self.cache.unspent = unspentList;
-    var uu = self.txProposals.getUsedUnspent(self.maxRejectCount());
+    return self.computeUnspent(self.cache.unspent, cb);
-
-    for (var i in unspentList) {
-      var u = unspentList[i];
-      var name = u.txid + ',' + u.vout;
-      if (!uu[name] && (self.spendUnconfirmed || u.confirmations >= 1))
-        safeUnspendList.push(u);
-    }
-
-    return cb(null, safeUnspendList, unspentList);
   });
 };
 
+/**
+ *
+ * @callback getUnspentCallback
+ * @param {string} error
+ * @param {Object[]} safeUnspendList
+ * @param {Object[]} unspentList
+ */
+/**
+ * computeUnspent
+ *
+ * @param unspentList List of unprocessed utxos.
+ * @param {getUnspentCallback} cb
+ */
+Wallet.prototype.computeUnspent = function(unspentList, cb) {
+  var self = this;
+
+  var safeUnspendList = [];
+  var uu = this.txProposals.getUsedUnspent(this.maxRejectCount());
+
+  _.each(unspentList, function(u) {
+    var name = u.txid + ',' + u.vout;
+    if (!uu[name] && (self.spendUnconfirmed || u.confirmations >= 1))
+      safeUnspendList.push(u);
+  });
+
+  return cb(null, safeUnspendList, unspentList);
+};
+
 /**
  * spend
  *
@@ -2227,8 +2294,6 @@ Wallet.prototype.spend = function(opts, cb) {
   preconditions.checkArgument(amountSat, 'no amount');
   preconditions.checkArgument(toAddress, 'no address');
 
-  // TODO no bajar los unspends de vuelta...
-  //
   this.getUnspent(function(err, safeUnspent) {
     if (err) {
       log.info(err);
@@ -2254,9 +2319,6 @@ Wallet.prototype.spend = function(opts, cb) {
     }
 
     log.debug('TXP Added: ', ntxid);
-
-    self.sendIndexes();
-
     // Needs only one signature? Broadcast it!
     if (!self.requiresMultipleSignatures())
       return self.issueTx(ntxid, cb);
@@ -2366,7 +2428,6 @@ Wallet.prototype._createTxProposal = function(toAddress, amountSat, comment, utx
     signWith: keys,
   });
 
-  console.log('[Wallet.js.2303]'); //TODO
   return txp;
 };
 
@@ -2391,7 +2452,9 @@ Wallet.prototype.updateIndexes = function(callback) {
   async.parallel(tasks, function(err) {
     if (err) callback(err);
     log.debug('Wallet:' + self.id + ' Indexes updated');
-    self._newAddresses();
+    self.clearUnspentCache();
+    self.subscribeToAddresses();
+    self.emitAndKeepAlive('newAddresses');
     callback();
   });
 };

js/services/balanceService.js

@@ -2,10 +2,11 @@
 var bitcore = require('bitcore');
 
 angular.module('copayApp.services')
-  .factory('balanceService', function($rootScope, $filter, rateService) {
+  .factory('balanceService', function($rootScope, $filter, $timeout, rateService) {
     var root = {};
     var _balanceCache = {};
     root.clearBalanceCache = function(w) {
+      w.clearUnspentCache();
       delete _balanceCache[w.getId()];
     };
 
@@ -63,7 +64,7 @@ angular.module('copayApp.services')
       w = w || $rootScope.wallet;
       if (!w || !w.isComplete()) return;
 
-      copay.logger.debug('Updating balance of:',  w.getName(), isFocused);
+      copay.logger.debug('Updating balance of:', w.getName(), isFocused);
       var wid = w.getId();
 
 
@@ -80,13 +81,17 @@ angular.module('copayApp.services')
 
       root._fetchBalance(w, function(err, res) {
         if (err) throw err;
-        w.balanceInfo=_balanceCache[wid] = res;
+        w.balanceInfo = _balanceCache[wid] = res;
         w.balanceInfo.updating = false;
 
         if (isFocused) {
           $rootScope.updatingBalance = false;
         }
-        if (cb) cb();
+        // we alwalys calltimeout because if balance is cached, we are still on the same
+        // execution path
+        if (cb) $timeout(function() {
+          return cb();
+        }, 1);
       });
     };
 

js/util/HTTP.js

@@ -18,6 +18,7 @@ module.exports = {
       _success: function() {;
       },
       _error: function(_, err) {
+        console.trace(err);
         throw err;
       }
     };

test/Wallet.js

@@ -687,6 +687,7 @@ describe('Wallet model', function() {
           amount: u
         }
       }));
+      w.clearUnspentCache();
       w.getBalance(function(err, balance, balanceByAddr, safeBalance) {
         balance.should.equal(c.balance);
         done();
@@ -821,7 +822,7 @@ describe('Wallet model', function() {
         should.exist(id);
         status.should.equal(Wallet.TX_PROPOSAL_SENT);
         w.sendTxProposal.calledOnce.should.equal(true);
-        w.sendIndexes.calledOnce.should.equal(true);
+        w.sendIndexes.calledOnce.should.equal(false);
         done();
       });
     });
@@ -861,7 +862,7 @@ describe('Wallet model', function() {
       }, function(err, id, status) {
         err.should.equal('error');
         w.sendTxProposal.calledOnce.should.equal(false);
-        w.sendIndexes.calledOnce.should.equal(true);
+        w.sendIndexes.calledOnce.should.equal(false);
         done();
       });
     });

test/util.http.js

@@ -75,15 +75,15 @@ describe('http utils', function() {
       done();
     });
   });
-  it('should get with default error', function() {
+  it('should get with default error', function(done) {
     xhr.error = 1;
-
     var ret = httpUtils.request({
       xhr: xhr,
       method: 'GET',
       url: 'http://test',
-    });
+    }).error(function(){
-    ret._error.should.throw()
+      done();
+    })
   });
 
 });