bump

BTCPayServer/BTCPayServer.csproj

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.0</TargetFramework>
-		<Version>1.0.1.26</Version>
+		<Version>1.0.1.29</Version>
     <NoWarn>NU1701</NoWarn>
   </PropertyGroup>
   <ItemGroup>

BTCPayServer/Controllers/StoresController.cs

@@ -312,10 +312,10 @@ namespace BTCPayServer.Controllers
             var stores = await _Repo.GetStoresByUserId(GetUserId());
             var balances = stores
                                 .Select(s => s.GetDerivationStrategies(_NetworkProvider)
-                                              .Select(d => (Wallet: _WalletProvider.GetWallet(d.Network),
-                                                            DerivationStrategy: d.DerivationStrategyBase))
+                                              .Select(d => ((Wallet: _WalletProvider.GetWallet(d.Network),
+                                                            DerivationStrategy: d.DerivationStrategyBase)))
                                               .Where(_ => _.Wallet != null)
-                                              .Select(async _ => (await _.Wallet.GetBalance(_.DerivationStrategy)).ToString() + " " + _.Wallet.Network.CryptoCode))
+                                              .Select(async _ => (await GetBalanceString(_)).ToString() + " " + _.Wallet.Network.CryptoCode))
                                 .ToArray();
 
             await Task.WhenAll(balances.SelectMany(_ => _));
@@ -333,6 +333,21 @@ namespace BTCPayServer.Controllers
             return View(result);
         }
 
+        private static async Task<string> GetBalanceString((BTCPayWallet Wallet, DerivationStrategyBase DerivationStrategy) _)
+        {
+            using (CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
+            {
+                try
+                {
+                    return (await _.Wallet.GetBalance(_.DerivationStrategy, cts.Token)).ToString();
+                }
+                catch
+                {
+                    return "--";
+                }
+            }
+        }
+
         [HttpGet]
         [Route("{storeId}/delete")]
         public async Task<IActionResult> DeleteStore(string storeId)

BTCPayServer/Events/TxOutReceivedEvent.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using NBitcoin;
+using NBXplorer.DerivationStrategy;
 
 namespace BTCPayServer.Events
 {
@@ -10,6 +11,7 @@ namespace BTCPayServer.Events
     {
         public BTCPayNetwork Network { get; set; }
         public Script ScriptPubKey { get; set; }
+        public DerivationStrategyBase DerivationStrategy { get; set; }
 
         public override string ToString()
         {

BTCPayServer/HostedServices/InvoiceWatcher.cs

@@ -29,9 +29,6 @@ namespace BTCPayServer.HostedServices
             {
 
             }
-
-            public Dictionary<BTCPayNetwork, KnownState> KnownStates { get; set; }
-            public Dictionary<BTCPayNetwork, KnownState> ModifiedKnownStates { get; set; } = new Dictionary<BTCPayNetwork, KnownState>();
             public InvoiceEntity Invoice { get; set; }
             public List<object> Events { get; set; } = new List<object>();
 
@@ -64,14 +61,15 @@ namespace BTCPayServer.HostedServices
         }
         CompositeDisposable leases = new CompositeDisposable();
 
-        async Task NotifyReceived(Script scriptPubKey, BTCPayNetwork network)
+        async Task NotifyReceived(Events.TxOutReceivedEvent evt)
         {
-            var invoice = await _InvoiceRepository.GetInvoiceIdFromScriptPubKey(scriptPubKey, network.CryptoCode);
-            if (invoice != null)
+            var invoiceId = await _InvoiceRepository.GetInvoiceIdFromScriptPubKey(evt.ScriptPubKey, evt.Network.CryptoCode);
+            if (invoiceId != null)
             {
-                String address = scriptPubKey.GetDestinationAddress(network.NBitcoinNetwork)?.ToString() ?? scriptPubKey.ToString();
-                Logs.PayServer.LogInformation($"{address} is mapping to invoice {invoice}");
-                _WatchRequests.Add(invoice);
+                String address = evt.ScriptPubKey.GetDestinationAddress(evt.Network.NBitcoinNetwork)?.ToString() ?? evt.ScriptPubKey.ToString();
+                _WalletProvider.GetWallet(evt.Network).InvalidateCache(evt.DerivationStrategy);
+                Logs.PayServer.LogInformation($"{address} is mapping to invoice {invoiceId}");
+                _WatchRequests.Add(invoiceId);
             }
         }
 
@@ -99,8 +97,7 @@ namespace BTCPayServer.HostedServices
                     var stateBefore = invoice.Status;
                     var updateContext = new UpdateInvoiceContext()
                     {
-                        Invoice = invoice,
-                        KnownStates = changes
+                        Invoice = invoice
                     };
                     await UpdateInvoice(updateContext).ConfigureAwait(false);
                     if (updateContext.Dirty)
@@ -116,11 +113,6 @@ namespace BTCPayServer.HostedServices
                         _EventAggregator.Publish(evt, evt.GetType());
                     }
 
-                    foreach (var modifiedKnownState in updateContext.ModifiedKnownStates)
-                    {
-                        changes.AddOrReplace(modifiedKnownState.Key, modifiedKnownState.Value);
-                    }
-
                     if (invoice.Status == "complete" ||
                        ((invoice.Status == "invalid" || invoice.Status == "expired") && invoice.MonitoringExpiration < DateTimeOffset.UtcNow))
                     {
@@ -165,8 +157,6 @@ namespace BTCPayServer.HostedServices
                 if (coins.TimestampedCoins.Length == 0)
                     continue;
                 bool dirtyAddress = false;
-                if (coins.State != null)
-                    context.ModifiedKnownStates.AddOrReplace(coins.Wallet.Network, coins.State);
                 var alreadyAccounted = new HashSet<OutPoint>(invoice.GetPayments(coins.Wallet.Network).Select(p => p.Outpoint));
 
                 foreach (var coin in coins.TimestampedCoins.Where(c => !alreadyAccounted.Contains(c.Coin.Outpoint)))
@@ -283,7 +273,7 @@ namespace BTCPayServer.HostedServices
                                               Strategy: d.DerivationStrategyBase))
                                 .Where(d => d.Wallet != null)
                                 .Select(d => (Network: d.Network,
-                                              Coins: d.Wallet.GetCoins(d.Strategy, context.KnownStates.TryGet(d.Network))))
+                                              Coins: d.Wallet.GetCoins(d.Strategy)))
                                 .Select(async d =>
                                 {
                                     var coins = await d.Coins;
@@ -442,7 +432,7 @@ namespace BTCPayServer.HostedServices
             _Loop = StartLoop(_Cts.Token);
 
             leases.Add(_EventAggregator.Subscribe<Events.NewBlockEvent>(async b => { await NotifyBlock(); }));
-            leases.Add(_EventAggregator.Subscribe<Events.TxOutReceivedEvent>(async b => { await NotifyReceived(b.ScriptPubKey, b.Network); }));
+            leases.Add(_EventAggregator.Subscribe<Events.TxOutReceivedEvent>(async b => { await NotifyReceived(b); }));
             leases.Add(_EventAggregator.Subscribe<Events.InvoiceEvent>(async b =>
             {
                 if (b.Name == "invoice_created")

BTCPayServer/HostedServices/NBXplorerListener.cs

@@ -25,11 +25,9 @@ namespace BTCPayServer.HostedServices
         private TaskCompletionSource<bool> _RunningTask;
         private CancellationTokenSource _Cts;
         NBXplorerDashboard _Dashboards;
-        TransactionCacheProvider _TxCache;
 
         public NBXplorerListener(ExplorerClientProvider explorerClients,
                                 NBXplorerDashboard dashboard,
-                                TransactionCacheProvider cacheProvider,
                                 InvoiceRepository invoiceRepository,
                                 EventAggregator aggregator, IApplicationLifetime lifetime)
         {
@@ -39,7 +37,6 @@ namespace BTCPayServer.HostedServices
             _ExplorerClients = explorerClients;
             _Aggregator = aggregator;
             _Lifetime = lifetime;
-            _TxCache = cacheProvider;
         }
 
         CompositeDisposable leases = new CompositeDisposable();
@@ -137,17 +134,16 @@ namespace BTCPayServer.HostedServices
                         switch (newEvent)
                         {
                             case NBXplorer.Models.NewBlockEvent evt:
-                                _TxCache.GetTransactionCache(network).NewBlock(evt.Hash, evt.PreviousBlockHash);
                                 _Aggregator.Publish(new Events.NewBlockEvent() { CryptoCode = evt.CryptoCode });
                                 break;
                             case NBXplorer.Models.NewTransactionEvent evt:
                                 foreach (var txout in evt.Outputs)
                                 {
-                                    _TxCache.GetTransactionCache(network).AddToCache(evt.TransactionData);
                                     _Aggregator.Publish(new Events.TxOutReceivedEvent()
                                     {
                                         Network = network,
-                                        ScriptPubKey = txout.ScriptPubKey
+                                        ScriptPubKey = txout.ScriptPubKey,
+                                        DerivationStrategy = txout.DerivationStrategy
                                     });
                                 }
                                 break;

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -142,8 +142,6 @@ namespace BTCPayServer.Hosting
                 BlockTarget = 20
             });
 
-            services.AddSingleton<TransactionCacheProvider>();
-
             services.AddSingleton<IHostedService, NBXplorerWaiters>();
             services.AddSingleton<IHostedService, NBXplorerListener>();
             services.AddSingleton<IHostedService, InvoiceNotificationManager>();

BTCPayServer/Services/TransactionCache.cs

@@ -1,96 +0,0 @@
-using System;
-using Microsoft.Extensions.Logging;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Caching.Memory;
-using Microsoft.Extensions.Options;
-using NBitcoin;
-using NBXplorer.Models;
-
-namespace BTCPayServer.Services
-{
-    public class TransactionCacheProvider
-    {
-        IOptions<MemoryCacheOptions> _Options;
-        public TransactionCacheProvider(IOptions<MemoryCacheOptions> options)
-        {
-            _Options = options;
-        }
-
-        ConcurrentDictionary<string, TransactionCache> _TransactionCaches = new ConcurrentDictionary<string, TransactionCache>();
-        public TransactionCache GetTransactionCache(BTCPayNetwork network)
-        {
-            if (network == null)
-                throw new ArgumentNullException(nameof(network));
-            return _TransactionCaches.GetOrAdd(network.CryptoCode, c => new TransactionCache(_Options, network));
-        }
-    }
-    public class TransactionCache : IDisposable
-    {
-        //IOptions<MemoryCacheOptions> _Options;
-        public TransactionCache(IOptions<MemoryCacheOptions> options, BTCPayNetwork network)
-        {
-            //if (network == null)
-            //    throw new ArgumentNullException(nameof(network));
-            //_Options = options;
-            //_MemoryCache = new MemoryCache(_Options);
-            //Network = network;
-        }
-
-        //uint256 _LastHash;
-        //int _ConfOffset;
-        //IMemoryCache _MemoryCache;
-        
-        public void NewBlock(uint256 newHash, uint256 previousHash)
-        {
-            //if (_LastHash != previousHash)
-            //{
-            //    var old = _MemoryCache;
-            //    _ConfOffset = 0;
-            //    _MemoryCache = new MemoryCache(_Options);
-            //    Thread.MemoryBarrier();
-            //    old.Dispose();
-            //}
-            //else
-            //    _ConfOffset++;
-            //_LastHash = newHash;
-        }
-
-        public TimeSpan CacheSpan { get; private set; } = TimeSpan.FromMinutes(60);
-
-        public BTCPayNetwork Network { get; private set; }
-
-        public void AddToCache(TransactionResult tx)
-        {
-            //Logging.Logs.PayServer.LogInformation($"ADD CACHE: {tx.Transaction.GetHash()} ({tx.Confirmations} conf)");
-            //_MemoryCache.Set(tx.Transaction.GetHash(), tx, DateTimeOffset.UtcNow + CacheSpan);
-        }
-
-
-        public TransactionResult GetTransaction(uint256 txId)
-        {
-            //var ok = _MemoryCache.TryGetValue(txId.ToString(), out object tx);
-            //Logging.Logs.PayServer.LogInformation($"GET CACHE: {txId} ({ok} plus {_ConfOffset})");
-
-            //var result = tx as TransactionResult;
-            //var confOffset = _ConfOffset;
-            //if (result != null && result.Confirmations > 0 && confOffset > 0)
-            //{
-            //    var serializer = new NBXplorer.Serializer(Network.NBitcoinNetwork);
-            //    result = serializer.ToObject<TransactionResult>(serializer.ToString(result));
-            //    result.Confirmations += confOffset;
-            //    result.Height += confOffset;
-            //}
-            //return result;
-            return null; // Does not work correctly yet
-        }
-
-        public void Dispose()
-        {
-            //_MemoryCache.Dispose();
-        }
-    }
-}

BTCPayServer/Services/Wallets/BTCPayWallet.cs

@@ -1,4 +1,5 @@
 using NBitcoin;
+using Microsoft.Extensions.Logging;
 using NBXplorer;
 using NBXplorer.DerivationStrategy;
 using System;
@@ -10,6 +11,8 @@ using BTCPayServer.Data;
 using System.Threading;
 using NBXplorer.Models;
 using Microsoft.Extensions.Caching.Memory;
+using BTCPayServer.Logging;
+using System.Collections.Concurrent;
 
 namespace BTCPayServer.Services.Wallets
 {
@@ -25,21 +28,22 @@ namespace BTCPayServer.Services.Wallets
             public Coin Coin { get; set; }
         }
         public TimestampedCoin[] TimestampedCoins { get; set; }
-        public KnownState State { get; set; }
         public DerivationStrategyBase Strategy { get; set; }
         public BTCPayWallet Wallet { get; set; }
     }
     public class BTCPayWallet
     {
         private ExplorerClient _Client;
-        private TransactionCache _Cache;
-        public BTCPayWallet(ExplorerClient client, TransactionCache cache, BTCPayNetwork network)
+        private IMemoryCache _MemoryCache;
+        public BTCPayWallet(ExplorerClient client, IMemoryCache memoryCache, BTCPayNetwork network)
         {
             if (client == null)
                 throw new ArgumentNullException(nameof(client));
+            if (memoryCache == null)
+                throw new ArgumentNullException(nameof(memoryCache));
             _Client = client;
             _Network = network;
-            _Cache = cache;
+            _MemoryCache = memoryCache;
         }
 
 
@@ -52,7 +56,7 @@ namespace BTCPayServer.Services.Wallets
             }
         }
 
-        public TimeSpan CacheSpan { get; private set; } = TimeSpan.FromMinutes(60);
+        public TimeSpan CacheSpan { get; private set; } = TimeSpan.FromMinutes(30);
 
         public async Task<BitcoinAddress> ReserveAddressAsync(DerivationStrategyBase derivationStrategy)
         {
@@ -91,25 +95,69 @@ namespace BTCPayServer.Services.Wallets
         {
             if (txId == null)
                 throw new ArgumentNullException(nameof(txId));
-            var tx = _Cache.GetTransaction(txId);
-            if (tx != null)
-                return tx;
-            tx = await _Client.GetTransactionAsync(txId, cancellation);
-            _Cache.AddToCache(tx);
+            var tx = await _Client.GetTransactionAsync(txId, cancellation);
             return tx;
         }
 
-        public async Task<NetworkCoins> GetCoins(DerivationStrategyBase strategy, KnownState state, CancellationToken cancellation = default(CancellationToken))
+        public void InvalidateCache(DerivationStrategyBase strategy)
         {
-            var changes = await _Client.GetUTXOsAsync(strategy, state?.PreviousCall, false, cancellation).ConfigureAwait(false);
+            _MemoryCache.Remove("CACHEDCOINS_" + strategy.ToString());
+        }
+        ConcurrentDictionary<string, TaskCompletionSource<UTXOChanges>> _FetchingUTXOs = new ConcurrentDictionary<string, TaskCompletionSource<UTXOChanges>>();
+
+
+        public async Task<NetworkCoins> GetCoins(DerivationStrategyBase strategy, CancellationToken cancellation = default(CancellationToken))
+        {
+            UTXOChanges changes = await GetUTXOChanges(strategy, cancellation);
+
             return new NetworkCoins()
             {
                 TimestampedCoins = changes.Confirmed.UTXOs.Concat(changes.Unconfirmed.UTXOs).Select(c => new NetworkCoins.TimestampedCoin() { Coin = c.AsCoin(), DateTime = c.Timestamp }).ToArray(),
-                State = new KnownState() { PreviousCall = changes },
                 Strategy = strategy,
                 Wallet = this
             };
         }
+        private async Task<UTXOChanges> GetUTXOChanges(DerivationStrategyBase strategy, CancellationToken cancellation)
+        {
+            var thisCompletionSource = new TaskCompletionSource<UTXOChanges>();
+            var completionSource = _FetchingUTXOs.GetOrAdd(strategy.ToString(), (s) => thisCompletionSource);
+            if (thisCompletionSource != completionSource)
+                return await completionSource.Task;
+            try
+            {
+                var utxos = await _MemoryCache.GetOrCreateAsync("CACHEDCOINS_" + strategy.ToString(), async entry =>
+                {
+                    var now = DateTimeOffset.UtcNow;
+                    UTXOChanges result = null;
+                    try
+                    {
+                        result = await _Client.GetUTXOsAsync(strategy, null, false, cancellation).ConfigureAwait(false);
+                    }
+                    catch
+                    {
+                        Logs.PayServer.LogError("Call to NBXplorer GetUTXOsAsync timed out, this should never happen, please report this issue to NBXplorer developers");
+                        throw;
+                    }
+                    var spentTime = DateTimeOffset.UtcNow - now;
+                    if (spentTime.TotalSeconds > 30)
+                    {
+                        Logs.PayServer.LogWarning($"NBXplorer took {(int)spentTime.TotalSeconds} seconds to reply, there is something wrong, please report this issue to NBXplorer developers");
+                    }
+                    entry.AbsoluteExpiration = DateTimeOffset.UtcNow + CacheSpan;
+                    return result;
+                });
+                completionSource.TrySetResult(utxos);
+            }
+            catch(Exception ex)
+            {
+                completionSource.TrySetException(ex);
+            }
+            finally
+            {
+                _FetchingUTXOs.TryRemove(strategy.ToString(), out var unused);
+            }
+            return await completionSource.Task;
+        }
 
         public Task<BroadcastResult[]> BroadcastTransactionsAsync(List<Transaction> transactions)
         {
@@ -117,9 +165,11 @@ namespace BTCPayServer.Services.Wallets
             return Task.WhenAll(tasks);
         }
 
+
+
         public async Task<(Coin[], Dictionary<Script, KeyPath>)> GetUnspentCoins(DerivationStrategyBase derivationStrategy, CancellationToken cancellation = default(CancellationToken))
         {
-            var changes = await _Client.GetUTXOsAsync(derivationStrategy, null, false, cancellation).ConfigureAwait(false);
+            var changes = await GetUTXOChanges(derivationStrategy, cancellation);
             var keyPaths = new Dictionary<Script, KeyPath>();
             foreach (var coin in changes.GetUnspentUTXOs())
             {
@@ -128,10 +178,10 @@ namespace BTCPayServer.Services.Wallets
             return (changes.GetUnspentCoins(), keyPaths);
         }
 
-        public async Task<Money> GetBalance(DerivationStrategyBase derivationStrategy)
+        public async Task<Money> GetBalance(DerivationStrategyBase derivationStrategy, CancellationToken cancellation = default(CancellationToken))
         {
-            var result = await _Client.GetUTXOsAsync(derivationStrategy, null, true);
-            return result.GetUnspentUTXOs().Select(c => c.Value).Sum();
+            UTXOChanges changes = await GetUTXOChanges(derivationStrategy, cancellation);
+            return changes.GetUnspentUTXOs().Select(c => c.Value).Sum();
         }
     }
 }

BTCPayServer/Services/Wallets/BTCPayWalletProvider.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
 
 namespace BTCPayServer.Services.Wallets
 {
@@ -10,16 +11,16 @@ namespace BTCPayServer.Services.Wallets
     {
         private ExplorerClientProvider _Client;
         BTCPayNetworkProvider _NetworkProvider;
-        TransactionCacheProvider _TransactionCacheProvider;
+        IOptions<MemoryCacheOptions> _Options;
         public BTCPayWalletProvider(ExplorerClientProvider client,
-                                    TransactionCacheProvider transactionCacheProvider,
+                                    IOptions<MemoryCacheOptions> memoryCacheOption,
                                     BTCPayNetworkProvider networkProvider)
         {
             if (client == null)
                 throw new ArgumentNullException(nameof(client));
             _Client = client;
-            _TransactionCacheProvider = transactionCacheProvider;
             _NetworkProvider = networkProvider;
+            _Options = memoryCacheOption;
         }
 
         public BTCPayWallet GetWallet(BTCPayNetwork network)
@@ -36,7 +37,7 @@ namespace BTCPayServer.Services.Wallets
             var client = _Client.GetExplorerClient(cryptoCode);
             if (network == null || client == null)
                 return null;
-            return new BTCPayWallet(client, _TransactionCacheProvider.GetTransactionCache(network), network);
+            return new BTCPayWallet(client, new MemoryCache(_Options), network);
         }
 
         public bool IsAvailable(BTCPayNetwork network)