Batch saving Invoice Log events every 5 seconds

BTCPayServer/Controllers/InvoiceController.cs

@@ -44,6 +44,7 @@ namespace BTCPayServer.Controllers
         private readonly PaymentMethodHandlerDictionary _paymentMethodHandlerDictionary;
         private readonly ApplicationDbContextFactory _dbContextFactory;
         private readonly PullPaymentHostedService _paymentHostedService;
+        private readonly InvoiceLogsService _invoiceLogsService;
         readonly IServiceProvider _ServiceProvider;
         public InvoiceController(
             IServiceProvider serviceProvider,
@@ -57,7 +58,8 @@ namespace BTCPayServer.Controllers
             BTCPayNetworkProvider networkProvider,
             PaymentMethodHandlerDictionary paymentMethodHandlerDictionary,
             ApplicationDbContextFactory dbContextFactory,
-            PullPaymentHostedService paymentHostedService)
+            PullPaymentHostedService paymentHostedService,
+            InvoiceLogsService invoiceLogsService)
         {
             _ServiceProvider = serviceProvider;
             _CurrencyNameTable = currencyNameTable ?? throw new ArgumentNullException(nameof(currencyNameTable));
@@ -70,6 +72,7 @@ namespace BTCPayServer.Controllers
             _paymentMethodHandlerDictionary = paymentMethodHandlerDictionary;
             _dbContextFactory = dbContextFactory;
             _paymentHostedService = paymentHostedService;
+            _invoiceLogsService = invoiceLogsService;
             _CSP = csp;
         }
 
@@ -280,7 +283,7 @@ namespace BTCPayServer.Controllers
                 {
                     ex.Handle(e => { logs.Write($"Error while fetching rates {ex}", InvoiceEventData.EventSeverity.Error); return true; });
                 }
-                await _InvoiceRepository.AddInvoiceLogs(entity.Id, logs);
+                _invoiceLogsService.AddInvoiceLogs(entity.Id, logs);
             });
             _EventAggregator.Publish(new Events.InvoiceEvent(entity, InvoiceEvent.Created));
             return entity;

BTCPayServer/HostedServices/InvoiceLogsService.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Threading.Tasks;
+using BTCPayServer.Data;
+using BTCPayServer.Logging;
+using BTCPayServer.Services;
+using Microsoft.Extensions.Hosting;
+using NBitcoin;
+using NBitcoin.DataEncoders;
+
+namespace BTCPayServer.HostedServices
+{
+    public class InvoiceLogsService : IHostedService
+    {
+        private readonly ApplicationDbContextFactory _applicationDbContextFactory;
+        private readonly IBackgroundJobClient _backgroundJobClient;
+        private CancellationToken _cts;
+        public ConcurrentQueue<InvoiceEventData> QueuedLogs { get; set; } = new ConcurrentQueue<InvoiceEventData>();
+
+        public InvoiceLogsService(ApplicationDbContextFactory applicationDbContextFactory,
+            IBackgroundJobClient backgroundJobClient)
+        {
+            _applicationDbContextFactory = applicationDbContextFactory;
+            _backgroundJobClient = backgroundJobClient;
+        }
+
+        private async Task ProcessLogs(CancellationToken arg)
+        {
+            try
+            {
+                if (!QueuedLogs.IsEmpty)
+                {
+                    var cts = CancellationTokenSource.CreateLinkedTokenSource(arg, _cts);
+                    await using var context = _applicationDbContextFactory.CreateContext();
+                    while (QueuedLogs.TryDequeue(out var log))
+                    {
+                        await context.InvoiceEvents.AddAsync(log, cts.Token);
+                    }
+
+                    await context.SaveChangesAsync(cts.Token);
+                }
+            }
+            catch
+            {
+                // ignored
+            }
+
+            _backgroundJobClient.Schedule(ProcessLogs, TimeSpan.FromSeconds(5));
+        }
+
+        public void AddInvoiceLogs(string invoiceId, InvoiceLogs logs)
+        {
+            foreach (var log in logs.ToList())
+            {
+                QueuedLogs.Enqueue(new InvoiceEventData()
+                {
+                    Severity = log.Severity,
+                    InvoiceDataId = invoiceId,
+                    Message = log.Log,
+                    Timestamp = log.Timestamp,
+                    UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
+                });
+            }
+        }
+
+        public Task StartAsync(CancellationToken cancellationToken)
+        {
+            _cts = cancellationToken;
+            _backgroundJobClient.Schedule(ProcessLogs, TimeSpan.Zero);
+            return Task.CompletedTask;
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            return Task.CompletedTask;
+        }
+    }
+}

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -103,6 +103,8 @@ namespace BTCPayServer.Hosting
                     Directory.CreateDirectory(dbpath);
                 return new InvoiceRepository(dbContext, dbpath, o.GetRequiredService<BTCPayNetworkProvider>(), o.GetService<EventAggregator>());
             });
+            services.AddSingleton<InvoiceLogsService>();
+            services.AddSingleton<IHostedService>(provider => provider.GetService<InvoiceLogsService>() );
             services.AddSingleton<BTCPayServerEnvironment>();
             services.TryAddSingleton<TokenRepository>();
             services.TryAddSingleton<WalletRepository>();

BTCPayServer/Payments/Lightning/LightningListener.cs

@@ -9,6 +9,7 @@ using AngleSharp.Dom.Events;
 using BTCPayServer.Client.Models;
 using BTCPayServer.Data;
 using BTCPayServer.Events;
+using BTCPayServer.HostedServices;
 using BTCPayServer.Lightning;
 using BTCPayServer.Logging;
 using BTCPayServer.Services;
@@ -30,6 +31,7 @@ namespace BTCPayServer.Payments.Lightning
         private readonly LightningClientFactoryService lightningClientFactory;
         private readonly LightningLikePaymentHandler _lightningLikePaymentHandler;
         private readonly StoreRepository _storeRepository;
+        private readonly InvoiceLogsService _invoiceLogsService;
         readonly Channel<string> _CheckInvoices = Channel.CreateUnbounded<string>();
         Task _CheckingInvoice;
         readonly Dictionary<(string, string), LightningInstanceListener> _InstanceListeners = new Dictionary<(string, string), LightningInstanceListener>();
@@ -40,7 +42,8 @@ namespace BTCPayServer.Payments.Lightning
                               BTCPayNetworkProvider networkProvider,
                               LightningClientFactoryService lightningClientFactory,
                               LightningLikePaymentHandler lightningLikePaymentHandler,
-                              StoreRepository storeRepository)
+                              StoreRepository storeRepository,
+                              InvoiceLogsService invoiceLogsService)
         {
             _Aggregator = aggregator;
             _InvoiceRepository = invoiceRepository;
@@ -49,6 +52,7 @@ namespace BTCPayServer.Payments.Lightning
             this.lightningClientFactory = lightningClientFactory;
             _lightningLikePaymentHandler = lightningLikePaymentHandler;
             _storeRepository = storeRepository;
+            _invoiceLogsService = invoiceLogsService;
         }
 
         async Task CheckingInvoice(CancellationToken cancellation)
@@ -233,8 +237,7 @@ namespace BTCPayServer.Payments.Lightning
                             InvoiceEventData.EventSeverity.Error);
                     }
                 }
-
-                await _InvoiceRepository.AddInvoiceLogs(invoice.Id, logs);
+                _invoiceLogsService.AddInvoiceLogs(invoice.Id, logs);
                 _CheckInvoices.Writer.TryWrite(invoice.Id);
             }
 

BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs

@@ -89,6 +89,7 @@ namespace BTCPayServer.Payments.PayJoin
         private readonly DelayedTransactionBroadcaster _broadcaster;
         private readonly WalletRepository _walletRepository;
         private readonly BTCPayServerEnvironment _env;
+        private readonly InvoiceLogsService _invoiceLogsService;
 
         public PayJoinEndpointController(BTCPayNetworkProvider btcPayNetworkProvider,
             InvoiceRepository invoiceRepository, ExplorerClientProvider explorerClientProvider,
@@ -98,7 +99,8 @@ namespace BTCPayServer.Payments.PayJoin
             NBXplorerDashboard dashboard,
             DelayedTransactionBroadcaster broadcaster,
             WalletRepository walletRepository,
-            BTCPayServerEnvironment env)
+            BTCPayServerEnvironment env,
+            InvoiceLogsService invoiceLogsService)
         {
             _btcPayNetworkProvider = btcPayNetworkProvider;
             _invoiceRepository = invoiceRepository;
@@ -111,6 +113,7 @@ namespace BTCPayServer.Payments.PayJoin
             _broadcaster = broadcaster;
             _walletRepository = walletRepository;
             _env = env;
+            _invoiceLogsService = invoiceLogsService;
         }
 
         [HttpPost("")]
@@ -139,7 +142,7 @@ namespace BTCPayServer.Payments.PayJoin
                 });
             }
 
-            await using var ctx = new PayjoinReceiverContext(_invoiceRepository, _explorerClientProvider.GetExplorerClient(network), _payJoinRepository);
+            await using var ctx = new PayjoinReceiverContext(_invoiceRepository, _explorerClientProvider.GetExplorerClient(network), _payJoinRepository, _invoiceLogsService);
             ObjectResult CreatePayjoinErrorAndLog(int httpCode, PayjoinReceiverWellknownErrors err, string debug)
             {
                 ctx.Logs.Write($"Payjoin error: {debug}", InvoiceEventData.EventSeverity.Error);

BTCPayServer/Payments/PayJoin/PayjoinReceiverContext.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using BTCPayServer.HostedServices;
 using BTCPayServer.Logging;
 using BTCPayServer.Services.Invoices;
 using Microsoft.Extensions.Logging;
@@ -15,12 +16,14 @@ namespace BTCPayServer.Payments.PayJoin
         private readonly InvoiceRepository _invoiceRepository;
         private readonly ExplorerClient _explorerClient;
         private readonly PayJoinRepository _payJoinRepository;
+        private readonly InvoiceLogsService _invoiceLogsService;
 
-        public PayjoinReceiverContext(InvoiceRepository invoiceRepository, ExplorerClient explorerClient, PayJoinRepository payJoinRepository)
+        public PayjoinReceiverContext(InvoiceRepository invoiceRepository, ExplorerClient explorerClient, PayJoinRepository payJoinRepository, InvoiceLogsService invoiceLogsService)
         {
             _invoiceRepository = invoiceRepository;
             _explorerClient = explorerClient;
             _payJoinRepository = payJoinRepository;
+            _invoiceLogsService = invoiceLogsService;
         }
         public Invoice Invoice { get; set; }
         public NBitcoin.Transaction OriginalTransaction { get; set; }
@@ -31,7 +34,7 @@ namespace BTCPayServer.Payments.PayJoin
             List<Task> disposing = new List<Task>();
             if (Invoice != null)
             {
-                disposing.Add(_invoiceRepository.AddInvoiceLogs(Invoice.Id, Logs));
+                _invoiceLogsService.AddInvoiceLogs(Invoice.Id, Logs);
             }
             if (!doNotBroadcast && OriginalTransaction != null)
             {

BTCPayServer/Services/Invoices/InvoiceRepository.cs

@@ -217,23 +217,6 @@ retry:
             return temp.GetBlob(_Networks);
         }
 
-        public async Task AddInvoiceLogs(string invoiceId, InvoiceLogs logs)
-        {
-            await using var context = _ContextFactory.CreateContext();
-            foreach (var log in logs.ToList())
-            {
-                await context.InvoiceEvents.AddAsync(new InvoiceEventData()
-                {
-                    Severity = log.Severity,
-                    InvoiceDataId = invoiceId,
-                    Message = log.Log,
-                    Timestamp = log.Timestamp,
-                    UniqueId = Encoders.Hex.EncodeData(RandomUtils.GetBytes(10))
-                });
-            }
-            await context.SaveChangesAsync().ConfigureAwait(false);
-        }
-
         private string GetDestination(PaymentMethod paymentMethod)
         {
             // For legacy reason, BitcoinLikeOnChain is putting the hashes of addresses in database