Getting earlier transactions with pagination.

src/js/controllers/walletDetails.js

@@ -198,6 +198,7 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
     });
   };
 
+  
   var updateTxHistory = function(cb) {
     if (!cb) cb = function() {};
     $scope.vm.updateTxHistoryFailed = false;
@@ -234,6 +235,7 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
     });
   };
   
+
   function applyCurrencyAliases(txHistory) {
     var defaults = configService.getDefaults();
     var configCache = configService.getSync();
@@ -259,7 +261,7 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
   }
 
 
-  function updateTxHistoryUsingCachedData() {
+  function updateTxHistoryFromCachedData() {
     walletHistoryService.getCachedTxHistory($scope.wallet.id, function onGetCachedTxHistory(err, txHistory){
       $scope.vm.gettingCachedHistory = false;
       if (err) {
@@ -277,27 +279,24 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
       formatTxHistoryForDisplay(txHistory);
 
       completeTxHistory = txHistory;
-      showHistory();
+      showHistory(false);
       console.log('pagination Showing tx history items:', $scope.txHistory.length);
       $scope.$apply();
       console.log('pagination displayed cached history.');
     });
   }
 
-  function updateTxHistoryFromSmallCache(getLatest) {
+  function fetchAndShowTxHistory(getLatest, flushCacheOnNew) {
-    if (completeTxHistory.length > $scope.txHistory.length) {
+    $scope.vm.updatingTxHistory = true;
-      console.log('pagination Showing history we already have.');
-      currentTxHistoryDisplayPage++;
-      showHistory();
-      return
-    }
 
-    walletHistoryService.updateTxHistoryByPage($scope.wallet, getLatest, true, function onUpdateTxHistoryByPage(err, txHistory) {
+    walletHistoryService.updateLocalTxHistoryByPage($scope.wallet, getLatest, flushCacheOnNew, function onUpdateLocalTxHistoryByPage(err, txHistory) {
       console.log('pagination returned');
       $scope.vm.gettingInitialHistory = false;
+      $scope.vm.updatingTxHistory = false;
+      $scope.$broadcast('scroll.infiniteScrollComplete');
+
       if (err) {
         console.error('pagination Failed to get history.', err);
-        $scope.txHistory = null;
         $scope.vm.updateTxHistoryFailed = true;
         return;
       }
@@ -305,16 +304,17 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
       formatTxHistoryForDisplay(txHistory);
 
       completeTxHistory = txHistory;
-      showHistory();
+      showHistory(true);
       $scope.$apply();
     });
   }
 
   
-  function showHistory() {
+  function showHistory(showAll) {
     if (completeTxHistory) {
-      $scope.txHistory = completeTxHistory.slice(0, (currentTxHistoryDisplayPage + 1) * DISPLAY_PAGE_SIZE);
+      $scope.txHistory = showAll ? completeTxHistory : completeTxHistory.slice(0, (currentTxHistoryDisplayPage + 1) * DISPLAY_PAGE_SIZE);
       $scope.vm.allowInfiniteScroll = completeTxHistory.length > $scope.txHistory.length || !$scope.vm.gettingInitialHistory;
+      console.log('pagination Showing txs: ', $scope.txHistory.length);
     } else {
       $scope.vm.allowInfiniteScroll = false;
     }
@@ -372,6 +372,8 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
       $scope.$broadcast('scroll.infiniteScrollComplete');
       return;
     }
+
+    fetchAndShowTxHistory(false, false);
     /*
     $scope.vm.updatingTxHistory = true;
     $timeout(function() {
@@ -393,10 +395,11 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
     $scope.updateAll(true);
   };
 
-  $scope.updateAll = function(force, cb)  {
+  $scope.updateAll = function(forceStatusUpdate, getLatestTx, flushTxCacheOnNew)  {
-    updateStatus(force);
+    console.log('pagination updateAll()');
+    updateStatus(forceStatusUpdate);
     //updateTxHistory(cb);
-    //updateTxHistoryFromSmallCache();
+    fetchAndShowTxHistory(getLatestTx, flushTxCacheOnNew);
   };
 
   $scope.hideToggle = function() {
@@ -538,10 +541,11 @@ angular.module('copayApp.controllers').controller('walletDetailsController', fun
   var refreshInterval;
 
   $scope.$on("$ionicView.afterEnter", function(event, data) {
-    updateTxHistoryUsingCachedData();
+    updateTxHistoryFromCachedData();
-    $scope.updateAll();
+    $scope.updateAll(false, true, true);
     refreshAmountSection();
-    refreshInterval = $interval($scope.onRefresh, 10 * 1000);
+    //refreshInterval = $interval($scope.onRefresh, 10 * 1000);
+    //refreshInterval = $interval($scope.onRefresh, 120 * 1000); // For testing
   });
 
   $scope.$on("$ionicView.afterLeave", function(event, data) {

src/js/services/wallet-history.service.js

@@ -7,32 +7,109 @@
     .factory('walletHistoryService', walletHistoryService);
     
     function walletHistoryService(configService, storageService, lodash, $log, txFormatService) {
-      // 8 Transactions fit on an iPhone Plus screen when in Wallet Details.
+      //var PAGE_SIZE = 50;
-      // Make it a little bit bigger so it doesn't have to load more immediately
+      var PAGE_SIZE = 20; // For dev only
-      var PAGE_SIZE = 50 / 1.5;
       // How much to overlap on each end of the page, for mitigating inconsistent sort order.
-      var PAGE_OVERLAP_FRACTION = 0.5;
+      var PAGE_OVERLAP_FRACTION = 0.2;
+      var PAGE_OVERLAP = Math.floor(PAGE_SIZE * PAGE_OVERLAP_FRACTION);
+      // The amount of transactions in the new overlapping resultset that we already know about.
+      // If we know about at least this many, then there are probably no gaps.
+      var MIN_KNOWN_TX_OVERLAP = Math.floor(PAGE_OVERLAP * 0.5);
 
       var SAFE_CONFIRMATIONS = 6;
 
       var service = {
         getCachedTxHistory: getCachedTxHistory,
-        updateTxHistoryByPage: updateTxHistoryByPage
+        updateLocalTxHistoryByPage: updateLocalTxHistoryByPage,
       };
       return service;
 
+      function addEarlyTransactions(walletId, cachedTxs, newTxs) {
+
+        var cachedTxIds = {};
+        cachedTxs.forEach(function forCachedTx(tx){
+          cachedTxIds[tx.txid] = true;
+        });
+
+        var someTransactionWereNew = false;
+        var overlappingTxsCount = 0;
+
+        newTxs.forEach(function forNewTx(tx){
+          if (cachedTxIds[tx.txid]) {
+            overlappingTxsCount++;
+          } else {
+            someTransactionWereNew = true;
+            cachedTxs.push(tx);
+          }
+        });
+
+        console.log('pagination Overlapping transactions:', overlappingTxsCount);
+        if (overlappingTxsCount >= MIN_KNOWN_TX_OVERLAP) { // We are good
+          if (someTransactionWereNew) {
+            console.log('pagination someTransactionsWereNew');
+            saveTxHistory(walletId, cachedTxs);
+          }
+          return cachedTxs;
+        } else {
+          // We might be missing some txs.
+          console.error('We might be missing some txs in the history.');
+          // Our history is wrong, so remove it - we could instead, try to fetch data that was not so early.
+          storageService.removeTxHistory(walletId, function onRemoveTxHistory(){});
+          return [];
+        }
+
+      }
+
+      function addLatestTransactions(cachedTxs, newTxs) {
+      }
+
+      // Only clear the cache once we have received new transactions from the server.
+      /**
+       * @param {function(err, txs)} cb - transactions is always an array, may be empty
+       */
+      function fetchTxHistoryByPage(wallet, start, cb) {
+        var skip = Math.max(0, start - PAGE_OVERLAP);
+        var limit = PAGE_SIZE;
+
+        var opts = {
+          skip: skip,
+          limit: limit
+        };
+        wallet.getTxHistory(opts, function onTxHistory(err, txsFromServer) {
+          if (err) {
+            return cb(err, []);
+          }
+    
+          if (txsFromServer.length === 0) {
+            return cb(null, []);
+          }
+          console.log('pagination Transactions fetched:', txsFromServer.length);
+          
+          var processedTxs = processNewTxs(wallet, txsFromServer);
+
+          /*
+          if (getLatest) {
+            console.log('pagination Saving retrieved txs.');
+            saveTxHistory(wallet, processedTxs);
+          }
+          */
+
+          return cb(null, processedTxs);
+        });
+      }
+
       /**
        * @param {string} walletId
-       * @param {function(error, txs)} cb
+       * @param {function(error, txs)} cb - txs is always an array, may be empty
        */
       function getCachedTxHistory(walletId, cb) {
         storageService.getTxHistory(walletId, function onGetTxHistory(err, txHistoryString){
           if (err) {
-            return cb(err);
+            return cb(err, []);
           }
 
           if (!txHistoryString) {
-            return cb(null, txHistoryString);
+            return cb(null, []);
           }
 
           try {
@@ -40,7 +117,7 @@
             return cb(null, txHistory);
           } catch (e) {
             $log.error('Failed to parse tx history.', e);
-            return cb(e);
+            return cb(e, []);
           }
         });
       }
@@ -83,46 +160,64 @@
         return processedTxs;
       };
 
-      /**
+      function saveTxHistory(walletId, processedTxs) {
-       * A small cache of up to the 50 most recent transactions
+        console.log('pagination Saving transactions:', processedTxs.length);
-       */
-      function saveTxHistory(processedTxs, walletId) {
         storageService.setTxHistory(processedTxs, walletId, function onSetTxHistory(error){
-          // Looks like callback only gets called on error
+          if (error) {
-          $log.error('pagination Failed to save tx history.', error);
+            $log.error('pagination Failed to save tx history.', error);
-        });
+          } else {
-        
+            console.log('pagination Save successful.');
-      }
-
-      // Only clear the cache once we have received new transactions from the server.
-      function updateTxHistoryByPage(wallet, getLatest, flushCacheOnNew, cb) {
-        var skip = 0;
-        var limit = Math.floor(PAGE_SIZE * (1 + PAGE_OVERLAP_FRACTION));
-
-        var opts = {
-          skip: skip,
-          limit: limit
-        };
-        wallet.getTxHistory(opts, function onTxHistory(err, txsFromServer) {
-          if (err) {
-            return cb(err);
           }
-    
-          if (!txsFromServer.length) {
-            return cb(null, []);
-          }
-          
-          var processedTxs = processNewTxs(wallet, txsFromServer);
-
-          if (getLatest) {
-            console.log('pagination Saving retrieved txs.');
-            saveTxHistory(wallet, processedTxs);
-          }
-
-          return cb(null, processedTxs);
         });
       }
 
+
+      function updateLocalTxHistoryByPage(wallet, getLatest, flushCacheOnNew, cb) {
+
+        if (flushCacheOnNew) {
+          console.log('pagination Getting latest txs.');
+          fetchTxHistoryByPage(wallet, 0, function onFetchTxHistory(err, txs){
+            if (err) {
+              return cb(err, txs);
+            }
+            saveTxHistory(wallet.id, txs);
+            return cb(null, txs);
+          });
+        } else {
+          console.log('pagination Getting early txs.');
+          getCachedTxHistory(wallet.id, function onCachedHistory(err, cachedTxs){
+            if (err) {
+              $log.error('Failed to get cached tx history.', err);
+              return cb(err, []);
+            }
+
+            var start = getLatest ? 0 : cachedTxs.length;
+            console.log('pagination Transaction fetch start:', start);
+            fetchTxHistoryByPage(wallet, start, function onFetchHistory(err, fetchedTxs){
+                if (err) {
+                  return cb(err);
+                }
+
+                if (fetchedTxs.length === 0) {
+                  return cb(null, cachedTxs);
+                }
+
+                var txs = [];
+                if (getLatest) {
+                  txs = addLatestTransactions(wallet.id, cachedTxs, fetchedTxs);
+                } else {
+                  txs = addEarlyTransactions(wallet.id, cachedTxs, fetchedTxs);
+                }
+                return cb(null, txs);
+            });
+
+             
+          });
+        }
+      }
+
+      
+
     }