From ba2301ebfe14a703da9566fc8ee356c6b8d9f18d Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Thu, 19 Sep 2024 22:15:02 +0900 Subject: [PATCH 01/26] Refactor the InvoiceAddresses table (#6232) --- BTCPayServer.Common/BTCPayNetwork.cs | 2 +- BTCPayServer.Data/ApplicationDbContext.cs | 1 - BTCPayServer.Data/Data/AddressInvoiceData.cs | 3 +- BTCPayServer.Data/Data/InvoiceData.cs | 1 - BTCPayServer.Data/Data/PendingInvoiceData.cs | 18 ----- .../20240919085726_refactorinvoiceaddress.cs | 80 +++++++++++++++++++ .../ApplicationDbContextModelSnapshot.cs | 28 +------ BTCPayServer.Tests/DatabaseTests.cs | 20 +++++ BTCPayServer.Tests/TestUtils.cs | 2 +- BTCPayServer.Tests/UnitTest1.cs | 3 +- .../Controllers/UIInvoiceController.UI.cs | 8 +- .../Data/AddressInvoiceDataExtensions.cs | 31 ------- BTCPayServer/Data/InvoiceDataExtensions.cs | 2 +- .../Payments/Bitcoin/NBXplorerListener.cs | 9 +-- .../Payments/IPaymentMethodHandler.cs | 2 +- .../PayJoin/PayJoinEndpointController.cs | 4 +- .../Monero/Services/MoneroListener.cs | 5 +- .../Altcoins/Zcash/Services/ZcashListener.cs | 5 +- .../Services/Invoices/InvoiceEntity.cs | 2 +- .../Services/Invoices/InvoiceRepository.cs | 35 +++----- 20 files changed, 135 insertions(+), 126 deletions(-) delete mode 100644 BTCPayServer.Data/Data/PendingInvoiceData.cs create mode 100644 BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs delete mode 100644 BTCPayServer/Data/AddressInvoiceDataExtensions.cs diff --git a/BTCPayServer.Common/BTCPayNetwork.cs b/BTCPayServer.Common/BTCPayNetwork.cs index dde08a9ae4..c8e0ccc990 100644 --- a/BTCPayServer.Common/BTCPayNetwork.cs +++ b/BTCPayServer.Common/BTCPayNetwork.cs @@ -133,7 +133,7 @@ public virtual List FilterValidTransactions(List i.AddressInvoices).OnDelete(DeleteBehavior.Cascade); builder.Entity() #pragma warning disable CS0618 - .HasKey(o => o.Address); + .HasKey(o => new { o.PaymentMethodId, o.Address }); #pragma warning restore CS0618 } } diff --git a/BTCPayServer.Data/Data/InvoiceData.cs b/BTCPayServer.Data/Data/InvoiceData.cs index b03d0fe568..87768d33ae 100644 --- a/BTCPayServer.Data/Data/InvoiceData.cs +++ b/BTCPayServer.Data/Data/InvoiceData.cs @@ -26,7 +26,6 @@ public partial class InvoiceData : IHasBlobUntyped public string ExceptionStatus { get; set; } public List AddressInvoices { get; set; } public bool Archived { get; set; } - public List PendingInvoices { get; set; } public List InvoiceSearchData { get; set; } public List Refunds { get; set; } diff --git a/BTCPayServer.Data/Data/PendingInvoiceData.cs b/BTCPayServer.Data/Data/PendingInvoiceData.cs deleted file mode 100644 index a69647bd1f..0000000000 --- a/BTCPayServer.Data/Data/PendingInvoiceData.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Microsoft.EntityFrameworkCore; - -namespace BTCPayServer.Data -{ - public class PendingInvoiceData - { - public string Id { get; set; } - public InvoiceData InvoiceData { get; set; } - - internal static void OnModelCreating(ModelBuilder builder) - { - builder.Entity() - .HasOne(o => o.InvoiceData) - .WithMany(o => o.PendingInvoices) - .HasForeignKey(o => o.Id).OnDelete(DeleteBehavior.Cascade); - } - } -} diff --git a/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs new file mode 100644 index 0000000000..b179812289 --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs @@ -0,0 +1,80 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240919085726_refactorinvoiceaddress")] + public partial class refactorinvoiceaddress : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices"); + + migrationBuilder.AddColumn( + name: "PaymentMethodId", + table: "AddressInvoices", + type: "text", + nullable: false, + defaultValue: ""); + + migrationBuilder.Sql(""" + UPDATE "AddressInvoices" + SET + "Address" = (string_to_array("Address", '#'))[1], + "PaymentMethodId" = CASE WHEN (string_to_array("Address", '#'))[2] IS NULL THEN 'BTC-CHAIN' + WHEN STRPOS((string_to_array("Address", '#'))[2], '_') = 0 THEN (string_to_array("Address", '#'))[2] || '-CHAIN' + WHEN STRPOS((string_to_array("Address", '#'))[2], '_MoneroLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_MoneroLike','-CHAIN') + WHEN STRPOS((string_to_array("Address", '#'))[2], '_ZcashLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_ZcashLike','-CHAIN') + ELSE '' END; + + DELETE FROM "AddressInvoices" WHERE "PaymentMethodId" = ''; + """); + migrationBuilder.AddPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices", + columns: new[] { "PaymentMethodId", "Address" }); + migrationBuilder.Sql("VACUUM (ANALYZE) \"AddressInvoices\";", true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices"); + + migrationBuilder.DropColumn( + name: "PaymentMethodId", + table: "AddressInvoices"); + + migrationBuilder.AddPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices", + column: "Address"); + + migrationBuilder.CreateTable( + name: "PendingInvoices", + columns: table => new + { + Id = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_PendingInvoices", x => x.Id); + table.ForeignKey( + name: "FK_PendingInvoices_Invoices_Id", + column: x => x.Id, + principalTable: "Invoices", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + } + } +} diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index 39e6630c57..78bb8d5025 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -60,13 +60,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("BTCPayServer.Data.AddressInvoiceData", b => { + b.Property("PaymentMethodId") + .HasColumnType("text"); + b.Property("Address") .HasColumnType("text"); b.Property("InvoiceDataId") .HasColumnType("text"); - b.HasKey("Address"); + b.HasKey("PaymentMethodId", "Address"); b.HasIndex("InvoiceDataId"); @@ -634,16 +637,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("PayoutProcessors"); }); - modelBuilder.Entity("BTCPayServer.Data.PendingInvoiceData", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("PendingInvoices"); - }); - modelBuilder.Entity("BTCPayServer.Data.PlannedTransaction", b => { b.Property("Id") @@ -1331,17 +1324,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Store"); }); - modelBuilder.Entity("BTCPayServer.Data.PendingInvoiceData", b => - { - b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData") - .WithMany("PendingInvoices") - .HasForeignKey("Id") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("InvoiceData"); - }); - modelBuilder.Entity("BTCPayServer.Data.PullPaymentData", b => { b.HasOne("BTCPayServer.Data.StoreData", "StoreData") @@ -1572,8 +1554,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Payments"); - b.Navigation("PendingInvoices"); - b.Navigation("Refunds"); }); diff --git a/BTCPayServer.Tests/DatabaseTests.cs b/BTCPayServer.Tests/DatabaseTests.cs index af709d4bc8..f213fd827d 100644 --- a/BTCPayServer.Tests/DatabaseTests.cs +++ b/BTCPayServer.Tests/DatabaseTests.cs @@ -18,6 +18,26 @@ public DatabaseTests(ITestOutputHelper helper):base(helper) { } + [Fact] + public async Task CanMigrateInvoiceAddresses() + { + var tester = CreateDBTester(); + await tester.MigrateUntil("20240919085726_refactorinvoiceaddress"); + using var ctx = tester.CreateContext(); + var conn = ctx.Database.GetDbConnection(); + await conn.ExecuteAsync("INSERT INTO \"Invoices\" (\"Id\", \"Created\") VALUES ('i', NOW())"); + await conn.ExecuteAsync( + "INSERT INTO \"AddressInvoices\" VALUES ('aaa#BTC', 'i'),('bbb','i'),('ccc#BTC_LNU', 'i'),('ddd#XMR_MoneroLike', 'i'),('eee#ZEC_ZcashLike', 'i')"); + await tester.ContinueMigration(); + foreach (var v in new[] { ("aaa", "BTC-CHAIN"), ("bbb", "BTC-CHAIN"), ("ddd", "XMR-CHAIN") , ("eee", "ZEC-CHAIN") }) + { + var ok = await conn.ExecuteScalarAsync("SELECT 't'::BOOLEAN FROM \"AddressInvoices\" WHERE \"Address\"=@a AND \"PaymentMethodId\"=@b", new { a = v.Item1, b = v.Item2 }); + Assert.True(ok); + } + var notok = await conn.ExecuteScalarAsync("SELECT 't'::BOOLEAN FROM \"AddressInvoices\" WHERE \"Address\"='ccc'"); + Assert.False(notok); + } + [Fact] public async Task CanMigratePayoutsAndPullPayments() { diff --git a/BTCPayServer.Tests/TestUtils.cs b/BTCPayServer.Tests/TestUtils.cs index 09c3b0cc37..18589e1c7f 100644 --- a/BTCPayServer.Tests/TestUtils.cs +++ b/BTCPayServer.Tests/TestUtils.cs @@ -16,7 +16,7 @@ namespace BTCPayServer.Tests public static class TestUtils { #if DEBUG && !SHORT_TIMEOUT - public const int TestTimeout = 600_000; + public const int TestTimeout = 60_000; #else public const int TestTimeout = 90_000; #endif diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 7f87f9879d..947010f563 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -2449,9 +2449,10 @@ public async void CheckOnionlocationForNonOnionHtmlRequests() private static bool IsMapped(Invoice invoice, ApplicationDbContext ctx) { var h = BitcoinAddress.Create(invoice.BitcoinAddress, Network.RegTest).ScriptPubKey.Hash.ToString(); + var pmi = PaymentTypes.CHAIN.GetPaymentMethodId("BTC"); return (ctx.AddressInvoices.Where(i => i.InvoiceDataId == invoice.Id).ToArrayAsync().GetAwaiter() .GetResult()) - .Where(i => i.GetAddress() == h).Any(); + .Where(i => i.Address == h && i.PaymentMethodId == pmi.ToString()).Any(); } diff --git a/BTCPayServer/Controllers/UIInvoiceController.UI.cs b/BTCPayServer/Controllers/UIInvoiceController.UI.cs index 88989784f7..7afeec204d 100644 --- a/BTCPayServer/Controllers/UIInvoiceController.UI.cs +++ b/BTCPayServer/Controllers/UIInvoiceController.UI.cs @@ -649,9 +649,7 @@ IActionResult NotSupported(string err) if (derivationScheme is null) return NotSupported("This feature is only available to BTC wallets"); var btc = PaymentTypes.CHAIN.GetPaymentMethodId("BTC"); - var bumpableAddresses = (await GetAddresses(selectedItems)) - .Where(p => p.GetPaymentMethodId() == btc) - .Select(p => p.GetAddress()).ToHashSet(); + var bumpableAddresses = await GetAddresses(btc, selectedItems); var utxos = await explorer.GetUTXOsAsync(derivationScheme); var bumpableUTXOs = utxos.GetUnspentUTXOs().Where(u => u.Confirmations == 0 && bumpableAddresses.Contains(u.ScriptPubKey.Hash.ToString())).ToArray(); var parameters = new MultiValueDictionary(); @@ -673,10 +671,10 @@ IActionResult NotSupported(string err) return RedirectToAction(nameof(ListInvoices), new { storeId }); } - private async Task GetAddresses(string[] selectedItems) + private async Task> GetAddresses(PaymentMethodId paymentMethodId, string[] selectedItems) { using var ctx = _dbContextFactory.CreateContext(); - return await ctx.AddressInvoices.Where(i => selectedItems.Contains(i.InvoiceDataId)).ToArrayAsync(); + return new HashSet(await ctx.AddressInvoices.Where(i => i.PaymentMethodId == paymentMethodId.ToString() && selectedItems.Contains(i.InvoiceDataId)).Select(i => i.Address).ToArrayAsync()); } [HttpGet("i/{invoiceId}")] diff --git a/BTCPayServer/Data/AddressInvoiceDataExtensions.cs b/BTCPayServer/Data/AddressInvoiceDataExtensions.cs deleted file mode 100644 index 79041470af..0000000000 --- a/BTCPayServer/Data/AddressInvoiceDataExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using BTCPayServer.Payments; - -namespace BTCPayServer.Data -{ - public static class AddressInvoiceDataExtensions - { -#pragma warning disable CS0618 - public static string GetAddress(this AddressInvoiceData addressInvoiceData) - { - if (addressInvoiceData.Address == null) - return null; - var index = addressInvoiceData.Address.LastIndexOf("#", StringComparison.InvariantCulture); - if (index == -1) - return addressInvoiceData.Address; - return addressInvoiceData.Address.Substring(0, index); - } - public static PaymentMethodId GetPaymentMethodId(this AddressInvoiceData addressInvoiceData) - { - if (addressInvoiceData.Address == null) - return null; - var index = addressInvoiceData.Address.LastIndexOf("#", StringComparison.InvariantCulture); - // Legacy AddressInvoiceData does not have the paymentMethodId attached to the Address - if (index == -1) - return PaymentMethodId.Parse("BTC"); - ///////////////////////// - return PaymentMethodId.TryParse(addressInvoiceData.Address.Substring(index + 1)); - } -#pragma warning restore CS0618 - } -} diff --git a/BTCPayServer/Data/InvoiceDataExtensions.cs b/BTCPayServer/Data/InvoiceDataExtensions.cs index f9c0efe41b..5fb5d9ecdc 100644 --- a/BTCPayServer/Data/InvoiceDataExtensions.cs +++ b/BTCPayServer/Data/InvoiceDataExtensions.cs @@ -73,7 +73,7 @@ public static InvoiceEntity GetBlob(this InvoiceData invoiceData) entity.Status = state.Status; if (invoiceData.AddressInvoices != null) { - entity.AvailableAddressHashes = invoiceData.AddressInvoices.Select(a => a.GetAddress() + a.GetPaymentMethodId()).ToHashSet(); + entity.Addresses = invoiceData.AddressInvoices.Select(a => (PaymentMethodId.Parse(a.PaymentMethodId), a.Address)).ToHashSet(); } if (invoiceData.Refunds != null) { diff --git a/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs b/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs index ef2e203e9d..2c87cf48df 100644 --- a/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs +++ b/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs @@ -144,6 +144,7 @@ private async Task Listen(BTCPayWallet wallet) Logs.PayServer.LogInformation($"{network.CryptoCode}: {paymentCount} payments happened while offline"); Logs.PayServer.LogInformation($"Connected to WebSocket of NBXplorer ({network.CryptoCode})"); + var pmi = PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode); while (!_Cts.IsCancellationRequested) { var newEvent = await session.NextEventAsync(_Cts.Token).ConfigureAwait(false); @@ -163,13 +164,11 @@ private async Task Listen(BTCPayWallet wallet) foreach (var output in validOutputs) { var key = network.GetTrackedDestination(output.Item1.ScriptPubKey); - var invoice = (await _InvoiceRepository.GetInvoicesFromAddresses(new[] { key })) - .FirstOrDefault(); + var invoice = await _InvoiceRepository.GetInvoiceFromAddress(pmi, key); if (invoice != null) { var address = output.matchedOutput.Address ?? network.NBXplorerNetwork.CreateAddress(evt.DerivationStrategy, output.Item1.KeyPath, output.Item1.ScriptPubKey); - var pmi = PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode); var handler = _handlers[pmi]; var details = new BitcoinLikePaymentData(output.outPoint, evt.TransactionData.Transaction.RBF, output.matchedOutput.KeyPath); @@ -198,7 +197,6 @@ await ReceivedPayment(wallet, invoice, payment, await UpdatePaymentStates(wallet, invoice.Id); } } - } } @@ -406,8 +404,7 @@ private async Task FindPaymentViaPolling(BTCPayWallet wallet, BTCPayNetwork coins = await wallet.GetUnspentCoins(strategy); coinsPerDerivationStrategy.Add(strategy, coins); } - coins = coins.Where(c => invoice.AvailableAddressHashes.Contains(c.ScriptPubKey.Hash.ToString() + cryptoId)) - .ToArray(); + coins = coins.Where(c => invoice.Addresses.Contains((cryptoId, network.GetTrackedDestination(c.ScriptPubKey)))).ToArray(); foreach (var coin in coins.Where(c => !alreadyAccounted.Contains(c.OutPoint))) { var transaction = await wallet.GetTransactionAsync(coin.OutPoint.Hash); diff --git a/BTCPayServer/Payments/IPaymentMethodHandler.cs b/BTCPayServer/Payments/IPaymentMethodHandler.cs index 75c3842757..48c454bf3a 100644 --- a/BTCPayServer/Payments/IPaymentMethodHandler.cs +++ b/BTCPayServer/Payments/IPaymentMethodHandler.cs @@ -280,7 +280,7 @@ public PaymentMethodContext( /// /// This string can be used to query AddressInvoice to find the invoiceId /// - public List TrackedDestinations { get; } = new List(); + public List TrackedDestinations { get; } = new(); internal async Task BeforeFetchingRates() { diff --git a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs index 449d3c2ff4..41a910e7e2 100644 --- a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs +++ b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs @@ -265,8 +265,8 @@ ObjectResult CreatePayjoinErrorAndLog(int httpCode, PayjoinReceiverWellknownErro if (walletReceiveMatch is null) { - var key = output.ScriptPubKey.Hash + "#" + network.CryptoCode.ToUpperInvariant(); - invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { key })).FirstOrDefault(); + var key = network.GetTrackedDestination(output.ScriptPubKey); + invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, key); if (invoice is null) continue; accountDerivation = _handlers.GetDerivationStrategy(invoice, network); diff --git a/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs b/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs index 0a1bfaefcd..9c1e14a25a 100644 --- a/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs +++ b/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs @@ -284,12 +284,9 @@ private async Task OnTransactionUpdated(string cryptoCode, string transactionHas foreach (var destination in transfer.Transfers.GroupBy(destination => destination.Address)) { //find the invoice corresponding to this address, else skip - var address = destination.Key + "#" + paymentMethodId; - var invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { address })).FirstOrDefault(); + var invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, destination.Key); if (invoice == null) - { continue; - } var index = destination.First().SubaddrIndex; diff --git a/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs b/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs index c7193451eb..a9a02a9e40 100644 --- a/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs +++ b/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs @@ -282,12 +282,9 @@ private async Task OnTransactionUpdated(string cryptoCode, string transactionHas foreach (var destination in transfer.Transfers.GroupBy(destination => destination.Address)) { //find the invoice corresponding to this address, else skip - var address = destination.Key + "#" + paymentMethodId; - var invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { address })).FirstOrDefault(); + var invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, destination.Key); if (invoice == null) - { continue; - } var index = destination.First().SubaddrIndex; diff --git a/BTCPayServer/Services/Invoices/InvoiceEntity.cs b/BTCPayServer/Services/Invoices/InvoiceEntity.cs index 46f0d09230..e9f065db6d 100644 --- a/BTCPayServer/Services/Invoices/InvoiceEntity.cs +++ b/BTCPayServer/Services/Invoices/InvoiceEntity.cs @@ -495,7 +495,7 @@ private Uri FillPlaceholdersUri(string v) public DateTimeOffset MonitoringExpiration { get; set; } [JsonIgnore] - public HashSet AvailableAddressHashes { get; set; } + public HashSet<(PaymentMethodId PaymentMethodId, string Address)> Addresses { get; set; } [JsonProperty] public bool ExtendedNotifications { get; set; } diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index 584dd7d060..ce3c17c315 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -67,29 +67,16 @@ public InvoiceEntity CreateNewInvoice(string storeId) }; } - public async Task> GetInvoicesFromAddresses(string[] addresses) + public async Task GetInvoiceFromAddress(PaymentMethodId paymentMethodId, string address) { - if (addresses.Length is 0) - return Array.Empty(); using var db = _applicationDbContextFactory.CreateContext(); - if (addresses.Length == 1) - { - var address = addresses[0]; - return (await db.AddressInvoices - .Include(a => a.InvoiceData.Payments) - .Where(a => a.Address == address) - .Select(a => a.InvoiceData) - .ToListAsync()).Select(ToEntity); - } - else - { - return (await db.AddressInvoices - .Include(a => a.InvoiceData.Payments) - .Where(a => addresses.Contains(a.Address)) - .Select(a => a.InvoiceData) - .ToListAsync()).Select(ToEntity); - } - } + var row = (await db.AddressInvoices + .Include(a => a.InvoiceData.Payments) + .Where(a => a.PaymentMethodId == paymentMethodId.ToString() && a.Address == address) + .Select(a => a.InvoiceData) + .FirstOrDefaultAsync()); + return row is null ? null : ToEntity(row); + } public async Task GetInvoicesWithPendingPayments(PaymentMethodId paymentMethodId, bool includeAddresses = false) { @@ -190,7 +177,8 @@ public async Task CreateInvoiceAsync(InvoiceCreationContext creationContext) await context.AddressInvoices.AddAsync(new AddressInvoiceData() { InvoiceDataId = invoice.Id, - Address = trackedDestination + Address = trackedDestination, + PaymentMethodId = ctx.Key.ToString() }); } } @@ -311,7 +299,8 @@ public async Task NewPaymentPrompt(string invoiceId, PaymentMethodContext paymen await context.AddressInvoices.AddAsync(new AddressInvoiceData() { InvoiceDataId = invoiceId, - Address = tracked + Address = tracked, + PaymentMethodId = paymentPromptContext.PaymentMethodId.ToString() }); } AddToTextSearch(context, invoice, prompt.Destination); From f5e5174045f06c81cfef7871404a5ea4896e7cda Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Fri, 20 Sep 2024 18:54:36 +0900 Subject: [PATCH 02/26] Refactor: Add GetMonitoredInvoices to fetch pending invoices or those with pending payments (#6235) --- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../003.RefactorPendingInvoicesPayments.sql | 2 +- .../DBScripts/004.MonitoredInvoices.sql | 22 +++++ .../20240919034505_monitoredinvoices.cs | 15 ++++ BTCPayServer.Tests/UnitTest1.cs | 6 ++ BTCPayServer/HostedServices/InvoiceWatcher.cs | 12 ++- .../Payments/Bitcoin/NBXplorerListener.cs | 4 +- .../Payments/Lightning/LightningListener.cs | 90 ++++++++++--------- .../Monero/Services/MoneroListener.cs | 2 +- .../Altcoins/Zcash/Services/ZcashListener.cs | 2 +- .../Services/Invoices/InvoiceRepository.cs | 71 ++++++++++----- 11 files changed, 156 insertions(+), 71 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql create mode 100644 BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index cda071757a..0a3c73e521 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -21,5 +21,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/003.RefactorPendingInvoicesPayments.sql b/BTCPayServer.Data/DBScripts/003.RefactorPendingInvoicesPayments.sql index 68ac2d52b1..59d19c384f 100644 --- a/BTCPayServer.Data/DBScripts/003.RefactorPendingInvoicesPayments.sql +++ b/BTCPayServer.Data/DBScripts/003.RefactorPendingInvoicesPayments.sql @@ -5,5 +5,5 @@ $$ LANGUAGE sql IMMUTABLE; CREATE INDEX "IX_Invoices_Pending" ON "Invoices"((1)) WHERE is_pending("Status"); CREATE INDEX "IX_Payments_Pending" ON "Payments"((1)) WHERE is_pending("Status"); - DROP TABLE "PendingInvoices"; +ANALYZE "Invoices"; diff --git a/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql b/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql new file mode 100644 index 0000000000..b89f396ffc --- /dev/null +++ b/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql @@ -0,0 +1,22 @@ +CREATE OR REPLACE FUNCTION get_prompt(invoice_blob JSONB, payment_method_id TEXT) +RETURNS JSONB AS $$ + SELECT invoice_blob->'prompts'->payment_method_id +$$ LANGUAGE sql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id", p."Id" FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" +LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" +WHERE (p."Type" IS NOT NULL AND p."Type" = payment_method_id) OR + (p."Type" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs b/BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs new file mode 100644 index 0000000000..199b9070c7 --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs @@ -0,0 +1,15 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240919034505_monitoredinvoices")] + [DBScript("004.MonitoredInvoices.sql")] + public partial class monitoredinvoices : DBScriptsMigration + { + } +} diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 947010f563..88271c53dd 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -3157,6 +3157,12 @@ public async Task CanCreateReports() var invoiceId = GetInvoiceId(resp); await acc.PayOnChain(invoiceId); + // Quick unrelated test on GetMonitoredInvoices + var invoiceRepo = tester.PayTester.GetService(); + var monitored = Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN")), i => i.Id == invoiceId); + Assert.Single(monitored.Payments); + // + app = await client.CreatePointOfSaleApp(acc.StoreId, new PointOfSaleAppRequest { AppName = "Cart", diff --git a/BTCPayServer/HostedServices/InvoiceWatcher.cs b/BTCPayServer/HostedServices/InvoiceWatcher.cs index d4f07aa9ca..cc094a534a 100644 --- a/BTCPayServer/HostedServices/InvoiceWatcher.cs +++ b/BTCPayServer/HostedServices/InvoiceWatcher.cs @@ -14,6 +14,7 @@ using BTCPayServer.Services.Invoices; using BTCPayServer.Services.Notifications; using BTCPayServer.Services.Notifications.Blobs; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using NBitcoin; @@ -255,10 +256,19 @@ await _notificationSender.SendNotification(new StoreScope(b.Invoice.StoreId), private async Task WaitPendingInvoices() { - await Task.WhenAll((await _invoiceRepository.GetPendingInvoices()) + await Task.WhenAll((await GetPendingInvoices(_Cts.Token)) .Select(i => Wait(i)).ToArray()); } + private async Task GetPendingInvoices(CancellationToken cancellationToken) + { + using var ctx = _invoiceRepository.DbContextFactory.CreateContext(); + var rows = await ctx.Invoices.Where(i => Data.InvoiceData.IsPending(i.Status)) + .Select(o => o).ToArrayAsync(cancellationToken); + var invoices = rows.Select(_invoiceRepository.ToEntity).ToArray(); + return invoices; + } + async Task StartLoop(CancellationToken cancellation) { Logs.PayServer.LogInformation("Start watching invoices"); diff --git a/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs b/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs index 2c87cf48df..9a39b8518d 100644 --- a/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs +++ b/BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs @@ -235,7 +235,7 @@ await ReceivedPayment(wallet, invoice, payment, async Task UpdatePaymentStates(BTCPayWallet wallet) { - var invoices = await _InvoiceRepository.GetInvoicesWithPendingPayments(PaymentTypes.CHAIN.GetPaymentMethodId(wallet.Network.CryptoCode)); + var invoices = await _InvoiceRepository.GetMonitoredInvoices(PaymentTypes.CHAIN.GetPaymentMethodId(wallet.Network.CryptoCode)); await Task.WhenAll(invoices.Select(i => UpdatePaymentStates(wallet, i)).ToArray()); } async Task UpdatePaymentStates(BTCPayWallet wallet, string invoiceId, bool fireEvents = true) @@ -384,7 +384,7 @@ private async Task FindPaymentViaPolling(BTCPayWallet wallet, BTCPayNetwork { var handler = _handlers.GetBitcoinHandler(wallet.Network); int totalPayment = 0; - var invoices = await _InvoiceRepository.GetInvoicesWithPendingPayments(PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode), true); + var invoices = await _InvoiceRepository.GetMonitoredInvoices(PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode)); var coinsPerDerivationStrategy = new Dictionary(); foreach (var i in invoices) diff --git a/BTCPayServer/Payments/Lightning/LightningListener.cs b/BTCPayServer/Payments/Lightning/LightningListener.cs index 9cdeed7667..15efa0fc78 100644 --- a/BTCPayServer/Payments/Lightning/LightningListener.cs +++ b/BTCPayServer/Payments/Lightning/LightningListener.cs @@ -18,6 +18,7 @@ using BTCPayServer.Services; using BTCPayServer.Services.Invoices; using BTCPayServer.Services.Stores; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -71,66 +72,69 @@ public LightningListener(EventAggregator aggregator, bool needCheckOfflinePayments = true; async Task CheckingInvoice(CancellationToken cancellation) { -retry: - try + var pmis = _handlers.Where(h => h is LightningLikePaymentHandler).Select(handler => handler.PaymentMethodId).ToArray(); + foreach (var pmi in pmis) { - Logs.PayServer.LogInformation("Checking if any payment arrived on lightning while the server was offline..."); - foreach (var invoice in await _InvoiceRepository.GetPendingInvoices(cancellationToken: cancellation)) +retry: + try { - if (GetListenedInvoices(invoice).Count > 0) + Logs.PayServer.LogInformation("Checking if any payment arrived on lightning while the server was offline..."); + foreach (var invoice in await _InvoiceRepository.GetMonitoredInvoices(pmi, cancellation)) { - _CheckInvoices.Writer.TryWrite(invoice.Id); - _memoryCache.Set(GetCacheKey(invoice.Id), invoice, GetExpiration(invoice)); + if (GetListenedInvoices(invoice).Count > 0) + { + _CheckInvoices.Writer.TryWrite(invoice.Id); + _memoryCache.Set(GetCacheKey(invoice.Id), invoice, GetExpiration(invoice)); + } } - } - needCheckOfflinePayments = false; - Logs.PayServer.LogInformation("Processing lightning payments..."); + needCheckOfflinePayments = false; + Logs.PayServer.LogInformation("Processing lightning payments..."); - while (await _CheckInvoices.Reader.WaitToReadAsync(cancellation) && - _CheckInvoices.Reader.TryRead(out var invoiceId)) - { - var invoice = await GetInvoice(invoiceId); - - foreach (var listenedInvoice in GetListenedInvoices(invoice)) + while (await _CheckInvoices.Reader.WaitToReadAsync(cancellation) && + _CheckInvoices.Reader.TryRead(out var invoiceId)) { - var store = await GetStore(invoice.StoreId); - var lnConfig = _handlers.GetLightningConfig(store, listenedInvoice.Network); - if (lnConfig is null) - continue; - var connStr = GetLightningUrl(listenedInvoice.Network.CryptoCode, lnConfig); - if (connStr is null) - continue; - var instanceListenerKey = (listenedInvoice.Network.CryptoCode, connStr.ToString()); - lock (_InstanceListeners) + var invoice = await GetInvoice(invoiceId); + + foreach (var listenedInvoice in GetListenedInvoices(invoice)) { - if (!_InstanceListeners.TryGetValue(instanceListenerKey, out var instanceListener)) + var store = await GetStore(invoice.StoreId); + var lnConfig = _handlers.GetLightningConfig(store, listenedInvoice.Network); + if (lnConfig is null) + continue; + var connStr = GetLightningUrl(listenedInvoice.Network.CryptoCode, lnConfig); + if (connStr is null) + continue; + var instanceListenerKey = (listenedInvoice.Network.CryptoCode, connStr.ToString()); + lock (_InstanceListeners) { - instanceListener ??= new LightningInstanceListener(_InvoiceRepository, _Aggregator, lightningClientFactory, listenedInvoice.Network, _handlers, connStr, _paymentService, Logs); - _InstanceListeners.TryAdd(instanceListenerKey, instanceListener); + if (!_InstanceListeners.TryGetValue(instanceListenerKey, out var instanceListener)) + { + instanceListener ??= new LightningInstanceListener(_InvoiceRepository, _Aggregator, lightningClientFactory, listenedInvoice.Network, _handlers, connStr, _paymentService, Logs); + _InstanceListeners.TryAdd(instanceListenerKey, instanceListener); + } + instanceListener.AddListenedInvoice(listenedInvoice); + _ = instanceListener.PollPayment(listenedInvoice, cancellation); } - instanceListener.AddListenedInvoice(listenedInvoice); - _ = instanceListener.PollPayment(listenedInvoice, cancellation); } - } - if (_CheckInvoices.Reader.Count is 0) - this.CheckConnections(); + if (_CheckInvoices.Reader.Count is 0) + this.CheckConnections(); + } + } + catch when (cancellation.IsCancellationRequested) + { + } + catch (Exception ex) + { + await Task.Delay(1000, cancellation); + Logs.PayServer.LogWarning(ex, "Unhandled error in the LightningListener"); + goto retry; } - } - catch when (cancellation.IsCancellationRequested) - { - } - catch (Exception ex) - { - await Task.Delay(1000, cancellation); - Logs.PayServer.LogWarning(ex, "Unhandled error in the LightningListener"); - goto retry; } } - private string GetCacheKey(string invoiceId) { return $"{nameof(GetListenedInvoices)}-{invoiceId}"; diff --git a/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs b/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs index 9c1e14a25a..bc4342d058 100644 --- a/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs +++ b/BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs @@ -384,7 +384,7 @@ public static long ConfirmationsRequired(MoneroLikePaymentData details, SpeedPol private async Task UpdateAnyPendingMoneroLikePayment(string cryptoCode) { var paymentMethodId = PaymentTypes.CHAIN.GetPaymentMethodId(cryptoCode); - var invoices = await _invoiceRepository.GetInvoicesWithPendingPayments(paymentMethodId); + var invoices = await _invoiceRepository.GetMonitoredInvoices(paymentMethodId); if (!invoices.Any()) return; invoices = invoices.Where(entity => entity.GetPaymentPrompt(paymentMethodId)?.Activated is true).ToArray(); diff --git a/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs b/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs index a9a02a9e40..7ade159c4d 100644 --- a/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs +++ b/BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs @@ -374,7 +374,7 @@ public static int ConfirmationsRequired(SpeedPolicy speedPolicy) private async Task UpdateAnyPendingZcashLikePayment(string cryptoCode) { var paymentMethodId = PaymentTypes.CHAIN.GetPaymentMethodId(cryptoCode); - var invoices = await _invoiceRepository.GetInvoicesWithPendingPayments(paymentMethodId); + var invoices = await _invoiceRepository.GetMonitoredInvoices(paymentMethodId); if (!invoices.Any()) return; invoices = invoices.Where(entity => entity.GetPaymentPrompt(paymentMethodId).Activated).ToArray(); diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index ce3c17c315..b74a8ffcd6 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -34,7 +34,7 @@ static InvoiceRepository() private readonly ApplicationDbContextFactory _applicationDbContextFactory; private readonly EventAggregator _eventAggregator; - + public ApplicationDbContextFactory DbContextFactory => _applicationDbContextFactory; public InvoiceRepository(ApplicationDbContextFactory contextFactory, EventAggregator eventAggregator) { @@ -78,32 +78,59 @@ public async Task GetInvoiceFromAddress(PaymentMethodId paymentMe return row is null ? null : ToEntity(row); } - public async Task GetInvoicesWithPendingPayments(PaymentMethodId paymentMethodId, bool includeAddresses = false) + /// + /// Returns all invoices which either: + /// * Have the activated and are pending + /// * Aren't pending but have a payment from the that is pending + /// is filled with the monitored addresses of the for this invoice. + /// include the payments for this invoice. + /// + /// The payment method id + /// Cancellation token + /// + public async Task GetMonitoredInvoices(PaymentMethodId paymentMethodId, CancellationToken cancellationToken = default) { var pmi = paymentMethodId.ToString(); using var ctx = _applicationDbContextFactory.CreateContext(); - var invoiceIds = (await ctx.Payments.Where(p => PaymentData.IsPending(p.Status) && p.Type == pmi).Select(p => p.InvoiceDataId).ToArrayAsync()).Distinct().ToArray(); - if (invoiceIds.Length is 0) + var conn = ctx.Database.GetDbConnection(); + var rows = await conn.QueryAsync<(string Id, uint xmin, string[] addresses, string[] payments, string invoice)>(new(""" + SELECT + i."Id", + i.xmin, + array_agg(ai."Address") addresses, + COALESCE(array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL), '{}') as payments, + (array_agg(to_jsonb(i)))[1] as invoice + FROM get_monitored_invoices(@pmi) m + LEFT JOIN "Payments" p ON p."Id" = m.payment_id + LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id + LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" + WHERE ai."PaymentMethodId" = @pmi + GROUP BY i."Id"; + """ + , new { pmi = paymentMethodId.ToString() })); + if (Enumerable.TryGetNonEnumeratedCount(rows, out var c) && c == 0) return Array.Empty(); - return await GetInvoices(new InvoiceQuery() + List invoices = new List(); + foreach (var row in rows) { - InvoiceId = invoiceIds, - IncludeAddresses = true - }); - } - public async Task GetPendingInvoices(CancellationToken cancellationToken = default) - { - using var ctx = _applicationDbContextFactory.CreateContext(); - var rows = await ctx.Invoices.Where(i => InvoiceData.IsPending(i.Status)) - .Include(i => i.Payments) - .Select(o => o).ToArrayAsync(); - return rows.Select(ToEntity).ToArray(); - } - public async Task GetPendingInvoices() - { - using var ctx = _applicationDbContextFactory.CreateContext(); - return (await ctx.Invoices.Where(i => InvoiceData.IsPending(i.Status)).Select(i => i).ToArrayAsync()) - .Select(i => ToEntity(i)).ToArray(); + var jobj = JObject.Parse(row.invoice); + jobj.Remove("Blob"); + jobj["Blob2"] = jobj["Blob2"].ToString(Formatting.None); + var invoiceData = jobj.ToObject(); + invoiceData.XMin = row.xmin; + invoiceData.AddressInvoices = row.addresses.Select((a) => new AddressInvoiceData() { InvoiceDataId = invoiceData.Id, Address = a, PaymentMethodId = paymentMethodId.ToString() }).ToList(); + invoiceData.Payments = new List(); + foreach (var payment in row.payments) + { + jobj = JObject.Parse(payment); + jobj.Remove("Blob"); + jobj["Blob2"] = jobj["Blob2"].ToString(Formatting.None); + var paymentData = jobj.ToObject(); + invoiceData.Payments.Add(paymentData); + } + invoices.Add(ToEntity(invoiceData)); + } + return invoices.ToArray(); } public async Task> GetWebhookDeliveries(string invoiceId) From 36a5d0ee3f4cb73a0d1b43f8dd11c8faf19c7c1d Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sun, 22 Sep 2024 11:13:09 +0900 Subject: [PATCH 03/26] Fix: Monero and ZCash not tracking addresses --- .../Altcoins/Monero/Payments/MoneroLikePaymentMethodHandler.cs | 1 + .../Altcoins/Zcash/Payments/ZcashLikePaymentMethodHandler.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/BTCPayServer/Services/Altcoins/Monero/Payments/MoneroLikePaymentMethodHandler.cs b/BTCPayServer/Services/Altcoins/Monero/Payments/MoneroLikePaymentMethodHandler.cs index aa4705b616..1e216c3cce 100644 --- a/BTCPayServer/Services/Altcoins/Monero/Payments/MoneroLikePaymentMethodHandler.cs +++ b/BTCPayServer/Services/Altcoins/Monero/Payments/MoneroLikePaymentMethodHandler.cs @@ -80,6 +80,7 @@ public async Task ConfigurePrompt(PaymentMethodContext context) context.Prompt.Destination = address.Address; context.Prompt.PaymentMethodFee = MoneroMoney.Convert(feeRatePerByte * 100); context.Prompt.Details = JObject.FromObject(details, Serializer); + context.TrackedDestinations.Add(address.Address); } private MoneroPaymentPromptDetails ParsePaymentMethodConfig(JToken config) { diff --git a/BTCPayServer/Services/Altcoins/Zcash/Payments/ZcashLikePaymentMethodHandler.cs b/BTCPayServer/Services/Altcoins/Zcash/Payments/ZcashLikePaymentMethodHandler.cs index 623a67d53e..76714b2f36 100644 --- a/BTCPayServer/Services/Altcoins/Zcash/Payments/ZcashLikePaymentMethodHandler.cs +++ b/BTCPayServer/Services/Altcoins/Zcash/Payments/ZcashLikePaymentMethodHandler.cs @@ -73,6 +73,7 @@ public async Task ConfigurePrompt(PaymentMethodContext context) AddressIndex = address.AddressIndex, DepositAddress = address.Address }, Serializer); + context.TrackedDestinations.Add(address.Address); } object IPaymentMethodHandler.ParsePaymentPromptDetails(Newtonsoft.Json.Linq.JToken details) From 3cf1aa00fa5002432f5586531e3687d8406e5716 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Mon, 23 Sep 2024 17:06:56 +0900 Subject: [PATCH 04/26] Payments should use composite key (#6240) * Payments should use composite key * Invert PK for InvoiceAddress --- BTCPayServer.Data/ApplicationDbContext.cs | 2 +- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../DBScripts/005.PaymentsRenaming.sql | 17 ++++++++ BTCPayServer.Data/Data/AddressInvoiceData.cs | 2 +- .../Data/PaymentData.Migration.cs | 6 +-- BTCPayServer.Data/Data/PaymentData.cs | 6 ++- .../20240919085726_refactorinvoiceaddress.cs | 4 +- .../20240923065254_refactorpayments.cs | 39 +++++++++++++++++++ ...ces.cs => 20240923071444_temprefactor2.cs} | 19 ++++----- .../ApplicationDbContextModelSnapshot.cs | 14 +++---- BTCPayServer.Tests/TestAccount.cs | 4 +- .../TestData/InvoiceMigrationTestVectors.json | 8 ++-- .../Controllers/UIInvoiceController.UI.cs | 2 +- BTCPayServer/Data/PaymentDataExtensions.cs | 4 +- .../Services/Invoices/InvoiceRepository.cs | 4 +- .../Services/Invoices/PaymentService.cs | 2 +- 16 files changed, 98 insertions(+), 36 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql create mode 100644 BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs rename BTCPayServer.Data/Migrations/{20240405052858_cleanup_address_invoices.cs => 20240923071444_temprefactor2.cs} (51%) diff --git a/BTCPayServer.Data/ApplicationDbContext.cs b/BTCPayServer.Data/ApplicationDbContext.cs index 38c3f6e6a7..6bdb167183 100644 --- a/BTCPayServer.Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/ApplicationDbContext.cs @@ -82,7 +82,7 @@ protected override void OnModelCreating(ModelBuilder builder) PairingCodeData.OnModelCreating(builder); //PayjoinLock.OnModelCreating(builder); PaymentRequestData.OnModelCreating(builder, Database); - PaymentData.OnModelCreating(builder, Database); + PaymentData.OnModelCreating(builder); PayoutData.OnModelCreating(builder, Database); //PlannedTransaction.OnModelCreating(builder); PullPaymentData.OnModelCreating(builder, Database); diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index 0a3c73e521..fdece06d5a 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -22,5 +22,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql new file mode 100644 index 0000000000..b52c5e7420 --- /dev/null +++ b/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql @@ -0,0 +1,17 @@ +DROP FUNCTION get_monitored_invoices; +CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" +LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Data/AddressInvoiceData.cs b/BTCPayServer.Data/Data/AddressInvoiceData.cs index 77b8b86846..89fc2d37f0 100644 --- a/BTCPayServer.Data/Data/AddressInvoiceData.cs +++ b/BTCPayServer.Data/Data/AddressInvoiceData.cs @@ -19,7 +19,7 @@ internal static void OnModelCreating(ModelBuilder builder) .WithMany(i => i.AddressInvoices).OnDelete(DeleteBehavior.Cascade); builder.Entity() #pragma warning disable CS0618 - .HasKey(o => new { o.PaymentMethodId, o.Address }); + .HasKey(o => new { o.Address, o.PaymentMethodId }); #pragma warning restore CS0618 } } diff --git a/BTCPayServer.Data/Data/PaymentData.Migration.cs b/BTCPayServer.Data/Data/PaymentData.Migration.cs index 3c0397a180..8f9d29499a 100644 --- a/BTCPayServer.Data/Data/PaymentData.Migration.cs +++ b/BTCPayServer.Data/Data/PaymentData.Migration.cs @@ -44,9 +44,9 @@ public bool TryMigrate() } var cryptoCode = blob["cryptoCode"].Value(); - Type = cryptoCode + "_" + blob["cryptoPaymentDataType"].Value(); - Type = MigrationExtensions.MigratePaymentMethodId(Type); - var divisibility = MigrationExtensions.GetDivisibility(Type); + PaymentMethodId = cryptoCode + "_" + blob["cryptoPaymentDataType"].Value(); + PaymentMethodId = MigrationExtensions.MigratePaymentMethodId(PaymentMethodId); + var divisibility = MigrationExtensions.GetDivisibility(PaymentMethodId); Currency = blob["cryptoCode"].Value(); blob.Remove("cryptoCode"); blob.Remove("cryptoPaymentDataType"); diff --git a/BTCPayServer.Data/Data/PaymentData.cs b/BTCPayServer.Data/Data/PaymentData.cs index 74ab13ab59..76f2c52b1c 100644 --- a/BTCPayServer.Data/Data/PaymentData.cs +++ b/BTCPayServer.Data/Data/PaymentData.cs @@ -27,13 +27,15 @@ public partial class PaymentData : IHasBlobUntyped [Obsolete("Use Blob2 instead")] public byte[] Blob { get; set; } public string Blob2 { get; set; } - public string Type { get; set; } + public string PaymentMethodId { get; set; } [Obsolete("Use Status instead")] public bool? Accounted { get; set; } public PaymentStatus? Status { get; set; } public static bool IsPending(PaymentStatus? status) => throw new NotSupportedException(); - internal static void OnModelCreating(ModelBuilder builder, DatabaseFacade databaseFacade) + internal static void OnModelCreating(ModelBuilder builder) { + builder.Entity() + .HasKey(o => new { o.Id, o.PaymentMethodId }); builder.Entity() .HasOne(o => o.InvoiceData) .WithMany(i => i.Payments).OnDelete(DeleteBehavior.Cascade); diff --git a/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs index b179812289..7b34c9dae7 100644 --- a/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs +++ b/BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs @@ -33,13 +33,13 @@ WHEN STRPOS((string_to_array("Address", '#'))[2], '_') = 0 THEN (string_to_array WHEN STRPOS((string_to_array("Address", '#'))[2], '_MoneroLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_MoneroLike','-CHAIN') WHEN STRPOS((string_to_array("Address", '#'))[2], '_ZcashLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_ZcashLike','-CHAIN') ELSE '' END; - + ALTER TABLE "AddressInvoices" DROP COLUMN IF EXISTS "CreatedTime"; DELETE FROM "AddressInvoices" WHERE "PaymentMethodId" = ''; """); migrationBuilder.AddPrimaryKey( name: "PK_AddressInvoices", table: "AddressInvoices", - columns: new[] { "PaymentMethodId", "Address" }); + columns: new[] { "Address", "PaymentMethodId" }); migrationBuilder.Sql("VACUUM (ANALYZE) \"AddressInvoices\";", true); } diff --git a/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs new file mode 100644 index 0000000000..80ad50b00d --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs @@ -0,0 +1,39 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240923065254_refactorpayments")] + [DBScript("005.PaymentsRenaming.sql")] + public partial class refactorpayments : DBScriptsMigration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_Payments", + table: "Payments"); + + migrationBuilder.RenameColumn( + name: "Type", + table: "Payments", + newName: "PaymentMethodId"); + migrationBuilder.Sql("UPDATE \"Payments\" SET \"PaymentMethodId\"='' WHERE \"PaymentMethodId\" IS NULL;"); + migrationBuilder.AddPrimaryKey( + name: "PK_Payments", + table: "Payments", + columns: new[] { "Id", "PaymentMethodId" }); + base.Up(migrationBuilder); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + } + } +} diff --git a/BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs b/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs similarity index 51% rename from BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs rename to BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs index b5ac183830..6b34066222 100644 --- a/BTCPayServer.Data/Migrations/20240405052858_cleanup_address_invoices.cs +++ b/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs @@ -1,6 +1,4 @@ -using System; using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; @@ -9,17 +7,20 @@ namespace BTCPayServer.Migrations { [DbContext(typeof(ApplicationDbContext))] - [Migration("20240405052858_cleanup_address_invoices")] - public partial class cleanup_address_invoices : Migration + [Migration("20240923071444_temprefactor2")] + public partial class temprefactor2 : Migration { /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.Sql(@" -DELETE FROM ""AddressInvoices"" WHERE ""Address"" LIKE '%_LightningLike'; -ALTER TABLE ""AddressInvoices"" DROP COLUMN IF EXISTS ""CreatedTime""; -"); - migrationBuilder.Sql(@"VACUUM (FULL, ANALYZE) ""AddressInvoices"";", true); + migrationBuilder.DropPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices"); + + migrationBuilder.AddPrimaryKey( + name: "PK_AddressInvoices", + table: "AddressInvoices", + columns: new[] { "Address", "PaymentMethodId" }); } /// diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index 78bb8d5025..c9c22caf6b 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -60,16 +60,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("BTCPayServer.Data.AddressInvoiceData", b => { - b.Property("PaymentMethodId") + b.Property("Address") .HasColumnType("text"); - b.Property("Address") + b.Property("PaymentMethodId") .HasColumnType("text"); b.Property("InvoiceDataId") .HasColumnType("text"); - b.HasKey("PaymentMethodId", "Address"); + b.HasKey("Address", "PaymentMethodId"); b.HasIndex("InvoiceDataId"); @@ -482,6 +482,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .HasColumnType("text"); + b.Property("PaymentMethodId") + .HasColumnType("text"); + b.Property("Accounted") .HasColumnType("boolean"); @@ -506,10 +509,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Status") .HasColumnType("text"); - b.Property("Type") - .HasColumnType("text"); - - b.HasKey("Id"); + b.HasKey("Id", "PaymentMethodId"); b.HasIndex("InvoiceDataId"); diff --git a/BTCPayServer.Tests/TestAccount.cs b/BTCPayServer.Tests/TestAccount.cs index 90889ff8ea..72168ace80 100644 --- a/BTCPayServer.Tests/TestAccount.cs +++ b/BTCPayServer.Tests/TestAccount.cs @@ -698,11 +698,13 @@ public async Task ImportOldInvoices(string storeId = null) await writer.FlushAsync(); } isHeader = true; - using (var writer = db.BeginTextImport("COPY \"Payments\" (\"Id\",\"Blob\",\"InvoiceDataId\",\"Accounted\",\"Blob2\",\"Type\") FROM STDIN DELIMITER ',' CSV HEADER")) + using (var writer = db.BeginTextImport("COPY \"Payments\" (\"Id\",\"Blob\",\"InvoiceDataId\",\"Accounted\",\"Blob2\",\"PaymentMethodId\") FROM STDIN DELIMITER ',' CSV HEADER")) { foreach (var invoice in oldPayments) { var localPayment = invoice.Replace("3sgUCCtUBg6S8LJkrbdfAWbsJMqByFLfvSqjG6xKBWEd", storeId); + // Old data could have Type to null. + localPayment += "BTC-CHAIN"; await writer.WriteLineAsync(localPayment); } await writer.FlushAsync(); diff --git a/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json b/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json index 0623dbfcae..a3dda6d267 100644 --- a/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json +++ b/BTCPayServer.Tests/TestData/InvoiceMigrationTestVectors.json @@ -604,7 +604,7 @@ }, "expectedProperties": { "Created": "04/23/2019 18:27:56 +00:00", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Amount": "0.07299962", @@ -634,7 +634,7 @@ }, "expectedProperties": { "Created": "10/01/2018 14:13:22 +00:00", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Amount": "0.00017863", @@ -666,7 +666,7 @@ "Created": "03/21/2024 07:24:35 +00:00", "CreatedInMs": "1711005875969", "Amount": "0.00000001", - "Type": "BTC-LNURL", + "PaymentMethodId": "BTC-LNURL", "Currency": "BTC" } }, @@ -697,7 +697,7 @@ "Created": "03/20/2024 22:39:08 +00:00", "CreatedInMs": "1710974348741", "Amount": "0.00197864", - "Type": "BTC-CHAIN", + "PaymentMethodId": "BTC-CHAIN", "Currency": "BTC", "Status": "Settled", "Accounted": null diff --git a/BTCPayServer/Controllers/UIInvoiceController.UI.cs b/BTCPayServer/Controllers/UIInvoiceController.UI.cs index 7afeec204d..5416cce29b 100644 --- a/BTCPayServer/Controllers/UIInvoiceController.UI.cs +++ b/BTCPayServer/Controllers/UIInvoiceController.UI.cs @@ -674,7 +674,7 @@ IActionResult NotSupported(string err) private async Task> GetAddresses(PaymentMethodId paymentMethodId, string[] selectedItems) { using var ctx = _dbContextFactory.CreateContext(); - return new HashSet(await ctx.AddressInvoices.Where(i => i.PaymentMethodId == paymentMethodId.ToString() && selectedItems.Contains(i.InvoiceDataId)).Select(i => i.Address).ToArrayAsync()); + return new HashSet(await ctx.AddressInvoices.Where(i => selectedItems.Contains(i.InvoiceDataId) && i.PaymentMethodId == paymentMethodId.ToString()).Select(i => i.Address).ToArrayAsync()); } [HttpGet("i/{invoiceId}")] diff --git a/BTCPayServer/Data/PaymentDataExtensions.cs b/BTCPayServer/Data/PaymentDataExtensions.cs index de5302caec..cf001cff56 100644 --- a/BTCPayServer/Data/PaymentDataExtensions.cs +++ b/BTCPayServer/Data/PaymentDataExtensions.cs @@ -34,13 +34,13 @@ public static PaymentEntity SetBlob(this PaymentData paymentData, PaymentEntity } public static PaymentData SetBlob(this PaymentData paymentData, PaymentMethodId paymentMethodId, PaymentBlob blob) { - paymentData.Type = paymentMethodId.ToString(); + paymentData.PaymentMethodId = paymentMethodId.ToString(); paymentData.Blob2 = JToken.FromObject(blob, InvoiceDataExtensions.DefaultSerializer).ToString(Newtonsoft.Json.Formatting.None); return paymentData; } public static PaymentMethodId GetPaymentMethodId(this PaymentData paymentData) { - return PaymentMethodId.Parse(paymentData.Type); + return PaymentMethodId.Parse(paymentData.PaymentMethodId); } public static PaymentEntity GetBlob(this PaymentData paymentData) { diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index b74a8ffcd6..2088bd9778 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -72,7 +72,7 @@ public async Task GetInvoiceFromAddress(PaymentMethodId paymentMe using var db = _applicationDbContextFactory.CreateContext(); var row = (await db.AddressInvoices .Include(a => a.InvoiceData.Payments) - .Where(a => a.PaymentMethodId == paymentMethodId.ToString() && a.Address == address) + .Where(a => a.Address == address && a.PaymentMethodId == paymentMethodId.ToString()) .Select(a => a.InvoiceData) .FirstOrDefaultAsync()); return row is null ? null : ToEntity(row); @@ -101,7 +101,7 @@ public async Task GetMonitoredInvoices(PaymentMethodId paymentM COALESCE(array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL), '{}') as payments, (array_agg(to_jsonb(i)))[1] as invoice FROM get_monitored_invoices(@pmi) m - LEFT JOIN "Payments" p ON p."Id" = m.payment_id + LEFT JOIN "Payments" p ON p."Id" = m.payment_id AND p."PaymentMethodId" = m.payment_method_id LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" WHERE ai."PaymentMethodId" = @pmi diff --git a/BTCPayServer/Services/Invoices/PaymentService.cs b/BTCPayServer/Services/Invoices/PaymentService.cs index 1a04515c2d..3d591cd1d1 100644 --- a/BTCPayServer/Services/Invoices/PaymentService.cs +++ b/BTCPayServer/Services/Invoices/PaymentService.cs @@ -49,7 +49,7 @@ public async Task AddPayment(Data.PaymentData paymentData, HashSe if (invoice == null) return null; invoiceEntity = invoice.GetBlob(); - var pmi = PaymentMethodId.Parse(paymentData.Type); + var pmi = PaymentMethodId.Parse(paymentData.PaymentMethodId); PaymentPrompt paymentMethod = invoiceEntity.GetPaymentPrompt(pmi); if (paymentMethod is null || !_handlers.TryGetValue(pmi, out var handler)) return null; From 1d9ec253fb04d0f943f752af1c6ea4ed30cc0ed6 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Mon, 23 Sep 2024 23:59:18 +0900 Subject: [PATCH 05/26] Fix migration of Invoice's payment (#6241) --- .../Data/InvoiceData.Migration.cs | 2 +- .../Data/PaymentData.Migration.cs | 6 ++ BTCPayServer.Tests/TestAccount.cs | 2 +- .../BlobMigratorHostedService.cs | 83 ++++++++++--------- .../InvoiceBlobMigratorHostedService.cs | 6 ++ 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/BTCPayServer.Data/Data/InvoiceData.Migration.cs b/BTCPayServer.Data/Data/InvoiceData.Migration.cs index aea620db9d..843f712031 100644 --- a/BTCPayServer.Data/Data/InvoiceData.Migration.cs +++ b/BTCPayServer.Data/Data/InvoiceData.Migration.cs @@ -210,7 +210,7 @@ public bool TryMigrate() } blob.ConvertNumberToString("price"); - Currency = blob["currency"].Value(); + Currency = blob["currency"].Value().ToUpperInvariant(); var isTopup = blob["type"]?.Value() is "TopUp"; var amount = decimal.Parse(blob["price"].Value(), CultureInfo.InvariantCulture); Amount = isTopup && amount == 0 ? null : decimal.Parse(blob["price"].Value(), CultureInfo.InvariantCulture); diff --git a/BTCPayServer.Data/Data/PaymentData.Migration.cs b/BTCPayServer.Data/Data/PaymentData.Migration.cs index 8f9d29499a..63fb86e61d 100644 --- a/BTCPayServer.Data/Data/PaymentData.Migration.cs +++ b/BTCPayServer.Data/Data/PaymentData.Migration.cs @@ -1,11 +1,13 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.ComponentModel.DataAnnotations.Schema; using System.Linq; using System.Reflection.Metadata; using System.Text; using System.Threading.Tasks; using BTCPayServer.Migrations; +using Microsoft.EntityFrameworkCore.ChangeTracking.Internal; using NBitcoin; using NBitcoin.Altcoins; using NBitcoin.DataEncoders; @@ -44,6 +46,7 @@ public bool TryMigrate() } var cryptoCode = blob["cryptoCode"].Value(); + MigratedPaymentMethodId = PaymentMethodId; PaymentMethodId = cryptoCode + "_" + blob["cryptoPaymentDataType"].Value(); PaymentMethodId = MigrationExtensions.MigratePaymentMethodId(PaymentMethodId); var divisibility = MigrationExtensions.GetDivisibility(PaymentMethodId); @@ -163,6 +166,9 @@ public bool TryMigrate() } [NotMapped] public bool Migrated { get; set; } + [NotMapped] + [EditorBrowsable(EditorBrowsableState.Never)] + public string MigratedPaymentMethodId { get; set; } static readonly DateTimeOffset unixRef = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero); public static long DateTimeToMilliUnixTime(in DateTime time) diff --git a/BTCPayServer.Tests/TestAccount.cs b/BTCPayServer.Tests/TestAccount.cs index 72168ace80..a26f33f908 100644 --- a/BTCPayServer.Tests/TestAccount.cs +++ b/BTCPayServer.Tests/TestAccount.cs @@ -704,7 +704,7 @@ public async Task ImportOldInvoices(string storeId = null) { var localPayment = invoice.Replace("3sgUCCtUBg6S8LJkrbdfAWbsJMqByFLfvSqjG6xKBWEd", storeId); // Old data could have Type to null. - localPayment += "BTC-CHAIN"; + localPayment += "UNKNOWN"; await writer.WriteLineAsync(localPayment); } await writer.FlushAsync(); diff --git a/BTCPayServer/HostedServices/BlobMigratorHostedService.cs b/BTCPayServer/HostedServices/BlobMigratorHostedService.cs index e6221a0141..8a62bed948 100644 --- a/BTCPayServer/HostedServices/BlobMigratorHostedService.cs +++ b/BTCPayServer/HostedServices/BlobMigratorHostedService.cs @@ -29,7 +29,7 @@ internal class Settings public bool Complete { get; set; } } Task? _Migrating; - TaskCompletionSource _Cts = new TaskCompletionSource(); + CancellationTokenSource? _Cts; public BlobMigratorHostedService( ILogger logs, ISettingsRepository settingsRepository, @@ -46,7 +46,8 @@ public BlobMigratorHostedService( public Task StartAsync(CancellationToken cancellationToken) { - _Migrating = Migrate(cancellationToken); + _Cts = new CancellationTokenSource(); + _Migrating = Migrate(_Cts.Token); return Task.CompletedTask; } public int BatchSize { get; set; } = 1000; @@ -61,47 +62,57 @@ private async Task Migrate(CancellationToken cancellationToken) else Logs.LogInformation("Migrating from the beginning"); + int batchSize = BatchSize; +retry: while (!cancellationToken.IsCancellationRequested) { -retry: - List entities; - DateTimeOffset progress; - await using (var ctx = ApplicationDbContextFactory.CreateContext(o => o.CommandTimeout((int)TimeSpan.FromDays(1.0).TotalSeconds))) + try { - var query = GetQuery(ctx, settings?.Progress).Take(batchSize); - entities = await query.ToListAsync(cancellationToken); - if (entities.Count == 0) + List entities; + DateTimeOffset progress; + await using (var ctx = ApplicationDbContextFactory.CreateContext(o => o.CommandTimeout((int)TimeSpan.FromDays(1.0).TotalSeconds))) { - var count = await GetQuery(ctx, null).CountAsync(cancellationToken); - if (count != 0) + var query = GetQuery(ctx, settings?.Progress).Take(batchSize); + entities = await query.ToListAsync(cancellationToken); + if (entities.Count == 0) + { + var count = await GetQuery(ctx, null).CountAsync(cancellationToken); + if (count != 0) + { + settings = new Settings() { Progress = null }; + Logs.LogWarning("Corruption detected, reindexing the table..."); + await Reindex(ctx, cancellationToken); + goto retry; + } + await SettingsRepository.UpdateSetting(new Settings() { Complete = true }, SettingsKey); + Logs.LogInformation("Migration completed"); + await PostMigrationCleanup(ctx, cancellationToken); + return; + } + + try + { + progress = ProcessEntities(ctx, entities); + await ctx.SaveChangesAsync(); + batchSize = BatchSize; + } + catch (Exception ex) when (ex is DbUpdateConcurrencyException or TimeoutException or OperationCanceledException) { - settings = new Settings() { Progress = null }; - Logs.LogWarning("Corruption detected, reindexing the table..."); - await Reindex(ctx, cancellationToken); + batchSize /= 2; + batchSize = Math.Max(1, batchSize); goto retry; } - await SettingsRepository.UpdateSetting(new Settings() { Complete = true }, SettingsKey); - Logs.LogInformation("Migration completed"); - await PostMigrationCleanup(ctx, cancellationToken); - return; } - try - { - progress = ProcessEntities(ctx, entities); - await ctx.SaveChangesAsync(); - batchSize = BatchSize; - } - catch (Exception ex) when (ex is DbUpdateConcurrencyException or TimeoutException or OperationCanceledException) - { - batchSize /= 2; - batchSize = Math.Max(1, batchSize); - goto retry; - } + settings = new Settings() { Progress = progress }; + await SettingsRepository.UpdateSetting(settings, SettingsKey); + } + catch (Exception ex) + { + Logs.LogError(ex, "Error while migrating"); + goto retry; } - settings = new Settings() { Progress = progress }; - await SettingsRepository.UpdateSetting(settings, SettingsKey); } } @@ -121,11 +132,7 @@ public async Task IsComplete() public Task StopAsync(CancellationToken cancellationToken) { - _Cts.TrySetCanceled(); - return (_Migrating ?? Task.CompletedTask).ContinueWith(t => - { - if (t.IsFaulted) - Logs.LogError(t.Exception, "Error while migrating"); - }); + _Cts?.Cancel(); + return _Migrating ?? Task.CompletedTask; } } diff --git a/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs b/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs index 27e9ed76e3..00c74d472b 100644 --- a/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs +++ b/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs @@ -71,6 +71,12 @@ protected override DateTimeOffset ProcessEntities(ApplicationDbContext ctx, List paymentEntity.Details = JToken.FromObject(handler.ParsePaymentDetails(paymentEntity.Details), handler.Serializer); } pay.SetBlob(paymentEntity); + + if (pay.PaymentMethodId != pay.MigratedPaymentMethodId) + { + ctx.Entry(pay).State = EntityState.Added; + ctx.Payments.Remove(new PaymentData() { Id = pay.Id, PaymentMethodId = pay.MigratedPaymentMethodId }); + } } } return invoices[^1].Created; From 25e360e17558407c73e1ee2b014f9b6ad03470aa Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 08:43:30 +0900 Subject: [PATCH 06/26] Allow listeners to retrieve invoices with nonActivated prompts --- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../DBScripts/006.PaymentsRenaming.sql | 18 +++++++++++++ .../20240924071444_temprefactor3.cs | 15 +++++++++++ BTCPayServer.Tests/GreenfieldAPITests.cs | 9 +++++++ BTCPayServer.Tests/UnitTest1.cs | 6 +++-- .../Services/Invoices/InvoiceRepository.cs | 25 ++++++++++++++++--- 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql create mode 100644 BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index fdece06d5a..f2042776f0 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -23,5 +23,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql new file mode 100644 index 0000000000..393b411800 --- /dev/null +++ b/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql @@ -0,0 +1,18 @@ +DROP FUNCTION get_monitored_invoices; +CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT, include_non_activated BOOLEAN) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" +LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND + (include_non_activated IS TRUE OR (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE)); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs b/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs new file mode 100644 index 0000000000..6884ded7d2 --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs @@ -0,0 +1,15 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240924071444_temprefactor3")] + [DBScript("006.PaymentsRenaming.sql")] + public partial class temprefactor3 : DBScriptsMigration + { + } +} diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index d7ae0e2701..c654c29011 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -14,6 +14,7 @@ using BTCPayServer.Lightning; using BTCPayServer.Models.InvoicingModels; using BTCPayServer.NTag424; +using BTCPayServer.Payments; using BTCPayServer.Payments.Lightning; using BTCPayServer.PayoutProcessors; using BTCPayServer.Plugins.PointOfSale.Controllers; @@ -2711,6 +2712,14 @@ await user.AssertHasWebhookEvent(WebhookEventType.In invoiceObject = await client.GetOnChainWalletObject(user.StoreId, "BTC", new OnChainWalletObjectId("invoice", invoice.Id), false); Assert.DoesNotContain(invoiceObject.Links.Select(l => l.Type), t => t == "address"); + // Check if we can get the monitored invoice + var invoiceRepo = tester.PayTester.GetService(); + var includeNonActivated = true; + Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), includeNonActivated), i => i.Id == invoice.Id); + includeNonActivated = false; + Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), includeNonActivated), i => i.Id == invoice.Id); + Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN")), i => i.Id == invoice.Id); + // paymentMethods = await client.GetInvoicePaymentMethods(store.Id, invoice.Id); Assert.Single(paymentMethods); diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 88271c53dd..814d971d16 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -3161,9 +3161,11 @@ public async Task CanCreateReports() var invoiceRepo = tester.PayTester.GetService(); var monitored = Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN")), i => i.Id == invoiceId); Assert.Single(monitored.Payments); - // + monitored = Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), true), i => i.Id == invoiceId); + Assert.Single(monitored.Payments); + // - app = await client.CreatePointOfSaleApp(acc.StoreId, new PointOfSaleAppRequest + app = await client.CreatePointOfSaleApp(acc.StoreId, new PointOfSaleAppRequest { AppName = "Cart", DefaultView = PosViewType.Cart, diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index 2088bd9778..7c06a1070a 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -88,11 +88,30 @@ public async Task GetInvoiceFromAddress(PaymentMethodId paymentMe /// The payment method id /// Cancellation token /// - public async Task GetMonitoredInvoices(PaymentMethodId paymentMethodId, CancellationToken cancellationToken = default) + public Task GetMonitoredInvoices(PaymentMethodId paymentMethodId, CancellationToken cancellationToken = default) + => GetMonitoredInvoices(paymentMethodId, false, cancellationToken: cancellationToken); + + /// + /// Returns all invoices which either: + /// * Have the activated and are pending + /// * Aren't pending but have a payment from the that is pending + /// is filled with the monitored addresses of the for this invoice. + /// include the payments for this invoice. + /// + /// The payment method id + /// If true, include pending invoice with non activated payment methods + /// Cancellation token + /// + public async Task GetMonitoredInvoices(PaymentMethodId paymentMethodId, bool includeNonActivated, CancellationToken cancellationToken = default) { var pmi = paymentMethodId.ToString(); using var ctx = _applicationDbContextFactory.CreateContext(); var conn = ctx.Database.GetDbConnection(); + + string includeNonActivateQuery = String.Empty; + if (includeNonActivated) + includeNonActivateQuery = " AND (get_prompt(i.\"Blob2\", @pmi)->'activated')::BOOLEAN IS NOT FALSE)"; + var rows = await conn.QueryAsync<(string Id, uint xmin, string[] addresses, string[] payments, string invoice)>(new(""" SELECT i."Id", @@ -100,14 +119,14 @@ public async Task GetMonitoredInvoices(PaymentMethodId paymentM array_agg(ai."Address") addresses, COALESCE(array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL), '{}') as payments, (array_agg(to_jsonb(i)))[1] as invoice - FROM get_monitored_invoices(@pmi) m + FROM get_monitored_invoices(@pmi, @includeNonActivated) m LEFT JOIN "Payments" p ON p."Id" = m.payment_id AND p."PaymentMethodId" = m.payment_method_id LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" WHERE ai."PaymentMethodId" = @pmi GROUP BY i."Id"; """ - , new { pmi = paymentMethodId.ToString() })); + , new { pmi = paymentMethodId.ToString(), includeNonActivated })); if (Enumerable.TryGetNonEnumeratedCount(rows, out var c) && c == 0) return Array.Empty(); List invoices = new List(); From b726ef8a2e52b915d5c10a2141963e45aab2ff4d Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 09:43:02 +0900 Subject: [PATCH 07/26] Migrate PayoutProcessors's PayoutMethodId in entity migration --- .../Migrations/20240906010127_renamecol.cs | 9 +++++++++ BTCPayServer/Hosting/MigrationStartupTask.cs | 17 ----------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs b/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs index 1e698a9b99..0394f1bb12 100644 --- a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs +++ b/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs @@ -26,6 +26,15 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "PaymentMethod", table: "PayoutProcessors", newName: "PayoutMethodId"); + + migrationBuilder.Sql(""" + UPDATE "PayoutProcessors" + SET + "PaymentMethodId" = CASE WHEN STRPOS("PaymentMethodId", '_') = 0 THEN "PaymentMethodId" || '-CHAIN' + CASE WHEN STRPOS("PaymentMethodId", '_LightningLike') = 0 THEN "PaymentMethodId" || '-LN' + CASE WHEN STRPOS("PaymentMethodId", '_LNURLPAY') = 0 THEN "PaymentMethodId" || '-LN' + ELSE "PaymentMethodId" END; + """); } /// diff --git a/BTCPayServer/Hosting/MigrationStartupTask.cs b/BTCPayServer/Hosting/MigrationStartupTask.cs index 27201ba8b2..cdec5e1a1b 100644 --- a/BTCPayServer/Hosting/MigrationStartupTask.cs +++ b/BTCPayServer/Hosting/MigrationStartupTask.cs @@ -211,12 +211,6 @@ public async Task ExecuteAsync(CancellationToken cancellationToken = default) settings.MigrateToStoreConfig = true; await _Settings.UpdateSetting(settings); } - if (!settings.MigratePayoutProcessors) - { - await MigratePayoutProcessors(); - settings.MigratePayoutProcessors = true; - await _Settings.UpdateSetting(settings); - } } catch (Exception ex) { @@ -225,17 +219,6 @@ public async Task ExecuteAsync(CancellationToken cancellationToken = default) } } - private async Task MigratePayoutProcessors() - { - await using var ctx = _DBContextFactory.CreateContext(); - var processors = await ctx.PayoutProcessors.ToArrayAsync(); - foreach (var processor in processors) - { - processor.PayoutMethodId = processor.GetPayoutMethodId().ToString(); - } - await ctx.SaveChangesAsync(); - } - private async Task MigrateToStoreConfig() { await using var ctx = _DBContextFactory.CreateContext(); From 8a951940fd75b2763815e2d76841f4d38441c659 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 09:47:46 +0900 Subject: [PATCH 08/26] Remove dead property --- BTCPayServer/Services/MigrationSettings.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/BTCPayServer/Services/MigrationSettings.cs b/BTCPayServer/Services/MigrationSettings.cs index 59b5464067..69010903ef 100644 --- a/BTCPayServer/Services/MigrationSettings.cs +++ b/BTCPayServer/Services/MigrationSettings.cs @@ -31,6 +31,5 @@ public override string ToString() public bool FixMappedDomainAppType { get; set; } public bool MigrateAppYmlToJson { get; set; } public bool MigrateToStoreConfig { get; set; } - public bool MigratePayoutProcessors { get; internal set; } } } From 587d3aa612eb9d7c14a66da1cbd30310a4995f4b Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 09:52:28 +0900 Subject: [PATCH 09/26] Fix query --- BTCPayServer.Data/Migrations/20240906010127_renamecol.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs b/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs index 0394f1bb12..928b24310d 100644 --- a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs +++ b/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs @@ -30,10 +30,10 @@ protected override void Up(MigrationBuilder migrationBuilder) migrationBuilder.Sql(""" UPDATE "PayoutProcessors" SET - "PaymentMethodId" = CASE WHEN STRPOS("PaymentMethodId", '_') = 0 THEN "PaymentMethodId" || '-CHAIN' - CASE WHEN STRPOS("PaymentMethodId", '_LightningLike') = 0 THEN "PaymentMethodId" || '-LN' - CASE WHEN STRPOS("PaymentMethodId", '_LNURLPAY') = 0 THEN "PaymentMethodId" || '-LN' - ELSE "PaymentMethodId" END; + "PayoutMethodId" = CASE WHEN STRPOS("PayoutMethodId", '_') = 0 THEN "PayoutMethodId" || '-CHAIN' + WHEN STRPOS("PayoutMethodId", '_LightningLike') = 0 THEN "PayoutMethodId" || '-LN' + WHEN STRPOS("PayoutMethodId", '_LNURLPAY') = 0 THEN "PayoutMethodId" || '-LN' + ELSE "PayoutMethodId" END; """); } From fe48cd4236b51aa8171b2e15be14acf9fe568a4e Mon Sep 17 00:00:00 2001 From: Vincent Bouzon Date: Tue, 24 Sep 2024 08:44:51 +0200 Subject: [PATCH 10/26] fix InvoiceRepository.GetMonitoredInvoices (#6243) --- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../DBScripts/007.PaymentsRenaming.sql | 18 ++++++++++++++++++ .../Migrations/20240924071444_temprefactor4.cs | 15 +++++++++++++++ .../Services/Invoices/InvoiceRepository.cs | 6 ------ 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql create mode 100644 BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index f2042776f0..757cb9c49a 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -24,5 +24,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql new file mode 100644 index 0000000000..ae31a98536 --- /dev/null +++ b/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql @@ -0,0 +1,18 @@ +DROP FUNCTION get_monitored_invoices; +CREATE OR REPLACE FUNCTION get_monitored_invoices(arg_payment_method_id TEXT, include_non_activated BOOLEAN) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" +LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = arg_payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", arg_payment_method_id) IS NOT NULL AND + (include_non_activated IS TRUE OR (get_prompt(i."Blob2", arg_payment_method_id)->'activated')::BOOLEAN IS NOT FALSE)); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs b/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs new file mode 100644 index 0000000000..968a5c6ced --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs @@ -0,0 +1,15 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240924071444_temprefactor4")] + [DBScript("007.PaymentsRenaming.sql")] + public partial class temprefactor4 : DBScriptsMigration + { + } +} diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index 7c06a1070a..af2142a9be 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -104,14 +104,9 @@ public Task GetMonitoredInvoices(PaymentMethodId paymentMethodI /// public async Task GetMonitoredInvoices(PaymentMethodId paymentMethodId, bool includeNonActivated, CancellationToken cancellationToken = default) { - var pmi = paymentMethodId.ToString(); using var ctx = _applicationDbContextFactory.CreateContext(); var conn = ctx.Database.GetDbConnection(); - string includeNonActivateQuery = String.Empty; - if (includeNonActivated) - includeNonActivateQuery = " AND (get_prompt(i.\"Blob2\", @pmi)->'activated')::BOOLEAN IS NOT FALSE)"; - var rows = await conn.QueryAsync<(string Id, uint xmin, string[] addresses, string[] payments, string invoice)>(new(""" SELECT i."Id", @@ -123,7 +118,6 @@ FROM get_monitored_invoices(@pmi, @includeNonActivated) m LEFT JOIN "Payments" p ON p."Id" = m.payment_id AND p."PaymentMethodId" = m.payment_method_id LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" - WHERE ai."PaymentMethodId" = @pmi GROUP BY i."Id"; """ , new { pmi = paymentMethodId.ToString(), includeNonActivated })); From 9d3f8672d9364776d983a482faaed6e65d3d46b3 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 15:56:54 +0900 Subject: [PATCH 11/26] Fix GetMonitoredInvoices --- BTCPayServer.Data/BTCPayServer.Data.csproj | 1 + .../DBScripts/008.PaymentsRenaming.sql | 18 +++++++++++++ .../20240924081444_temprefactor5.cs | 15 +++++++++++ BTCPayServer.Tests/GreenfieldAPITests.cs | 4 +-- BTCPayServer.Tests/TestUtils.cs | 2 +- .../Services/Invoices/InvoiceRepository.cs | 26 +++++++++++++------ 6 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql create mode 100644 BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index 757cb9c49a..7e1dbd5f8a 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -25,5 +25,6 @@ + diff --git a/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql new file mode 100644 index 0000000000..c6e4439710 --- /dev/null +++ b/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql @@ -0,0 +1,18 @@ +DROP FUNCTION get_monitored_invoices; +CREATE OR REPLACE FUNCTION get_monitored_invoices(arg_payment_method_id TEXT, include_non_activated BOOLEAN) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ +WITH cte AS ( +-- Get all the invoices which are pending. Even if no payments. +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(i."Status") +UNION ALL +-- For invoices not pending, take all of those which have pending payments +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" + WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) +SELECT cte.* FROM cte +JOIN "Invoices" i ON cte.invoice_id=i."Id" +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_method_id=p."PaymentMethodId" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = arg_payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", arg_payment_method_id) IS NOT NULL AND + (include_non_activated IS TRUE OR (get_prompt(i."Blob2", arg_payment_method_id)->'inactive')::BOOLEAN IS NOT TRUE)); +$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs b/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs new file mode 100644 index 0000000000..f6efaab879 --- /dev/null +++ b/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs @@ -0,0 +1,15 @@ +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20240924081444_temprefactor5")] + [DBScript("008.PaymentsRenaming.sql")] + public partial class temprefactor5 : DBScriptsMigration + { + } +} diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index c654c29011..b7d791b41c 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -2717,8 +2717,8 @@ await user.AssertHasWebhookEvent(WebhookEventType.In var includeNonActivated = true; Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), includeNonActivated), i => i.Id == invoice.Id); includeNonActivated = false; - Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), includeNonActivated), i => i.Id == invoice.Id); - Assert.Single(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN")), i => i.Id == invoice.Id); + Assert.DoesNotContain(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN"), includeNonActivated), i => i.Id == invoice.Id); + Assert.DoesNotContain(await invoiceRepo.GetMonitoredInvoices(PaymentMethodId.Parse("BTC-CHAIN")), i => i.Id == invoice.Id); // paymentMethods = await client.GetInvoicePaymentMethods(store.Id, invoice.Id); diff --git a/BTCPayServer.Tests/TestUtils.cs b/BTCPayServer.Tests/TestUtils.cs index 18589e1c7f..1aebb0ae87 100644 --- a/BTCPayServer.Tests/TestUtils.cs +++ b/BTCPayServer.Tests/TestUtils.cs @@ -16,7 +16,7 @@ namespace BTCPayServer.Tests public static class TestUtils { #if DEBUG && !SHORT_TIMEOUT - public const int TestTimeout = 60_000; + public const int TestTimeout = 20_000; #else public const int TestTimeout = 90_000; #endif diff --git a/BTCPayServer/Services/Invoices/InvoiceRepository.cs b/BTCPayServer/Services/Invoices/InvoiceRepository.cs index af2142a9be..e43c63e343 100644 --- a/BTCPayServer/Services/Invoices/InvoiceRepository.cs +++ b/BTCPayServer/Services/Invoices/InvoiceRepository.cs @@ -108,17 +108,27 @@ public async Task GetMonitoredInvoices(PaymentMethodId paymentM var conn = ctx.Database.GetDbConnection(); var rows = await conn.QueryAsync<(string Id, uint xmin, string[] addresses, string[] payments, string invoice)>(new(""" + WITH invoices_payments AS ( SELECT - i."Id", - i.xmin, - array_agg(ai."Address") addresses, - COALESCE(array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL), '{}') as payments, - (array_agg(to_jsonb(i)))[1] as invoice + m.invoice_id, + array_agg(to_jsonb(p)) FILTER (WHERE p."Id" IS NOT NULL) as payments FROM get_monitored_invoices(@pmi, @includeNonActivated) m LEFT JOIN "Payments" p ON p."Id" = m.payment_id AND p."PaymentMethodId" = m.payment_method_id - LEFT JOIN "Invoices" i ON i."Id" = m.invoice_id - LEFT JOIN "AddressInvoices" ai ON i."Id" = ai."InvoiceDataId" - GROUP BY i."Id"; + GROUP BY 1 + ), + invoices_addresses AS ( + SELECT m.invoice_id, + array_agg(ai."Address") addresses + FROM get_monitored_invoices(@pmi, @includeNonActivated) m + JOIN "AddressInvoices" ai ON ai."InvoiceDataId" = m.invoice_id + WHERE ai."PaymentMethodId" = @pmi + GROUP BY 1 + ) + SELECT + ip.invoice_id, i.xmin, COALESCE(ia.addresses, '{}'), COALESCE(ip.payments, '{}'), to_jsonb(i) + FROM invoices_payments ip + JOIN "Invoices" i ON i."Id" = ip.invoice_id + LEFT JOIN invoices_addresses ia ON ia.invoice_id = ip.invoice_id; """ , new { pmi = paymentMethodId.ToString(), includeNonActivated })); if (Enumerable.TryGetNonEnumeratedCount(rows, out var c) && c == 0) From c97c9d4ece72ce8b42a1aff101bae4dc8b108ebb Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 22:07:02 +0900 Subject: [PATCH 12/26] Add SQL test for GetMonitoredInvoices --- BTCPayServer.Tests/DatabaseTester.cs | 25 +++-- BTCPayServer.Tests/DatabaseTests.cs | 106 +++++++++++++++++++++ BTCPayServer/Data/InvoiceDataExtensions.cs | 2 +- BTCPayServer/Data/PaymentDataExtensions.cs | 6 +- 4 files changed, 126 insertions(+), 13 deletions(-) diff --git a/BTCPayServer.Tests/DatabaseTester.cs b/BTCPayServer.Tests/DatabaseTester.cs index 19a3b98f92..9b373d3a20 100644 --- a/BTCPayServer.Tests/DatabaseTester.cs +++ b/BTCPayServer.Tests/DatabaseTester.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using BTCPayServer.Abstractions.Models; using BTCPayServer.Data; +using BTCPayServer.Services.Invoices; using BTCPayServer.Tests.Logging; using Dapper; using Microsoft.EntityFrameworkCore; @@ -42,6 +43,13 @@ public ApplicationDbContextFactory CreateContextFactory() }), _loggerFactory); } + public InvoiceRepository GetInvoiceRepository() + { + var logs = new BTCPayServer.Logging.Logs(); + logs.Configure(_loggerFactory); + return new InvoiceRepository(CreateContextFactory(), new EventAggregator(logs)); + } + public ApplicationDbContext CreateContext() => CreateContextFactory().CreateContext(); public async Task MigrateAsync() @@ -59,18 +67,21 @@ private async Task EnsureCreatedAsync() await conn.ExecuteAsync($"CREATE DATABASE \"{dbname}\";"); } - public async Task MigrateUntil(string migration) + public async Task MigrateUntil(string migration = null) { using var ctx = CreateContext(); var db = ctx.Database.GetDbConnection(); await EnsureCreatedAsync(); var migrations = ctx.Database.GetMigrations().ToArray(); - var untilMigrationIdx = Array.IndexOf(migrations, migration); - if (untilMigrationIdx == -1) - throw new InvalidOperationException($"Migration {migration} not found"); - notAppliedMigrations = migrations[untilMigrationIdx..]; - await db.ExecuteAsync("CREATE TABLE IF NOT EXISTS \"__EFMigrationsHistory\" (\"MigrationId\" TEXT, \"ProductVersion\" TEXT)"); - await db.ExecuteAsync("INSERT INTO \"__EFMigrationsHistory\" VALUES (@migration, '8.0.0')", notAppliedMigrations.Select(m => new { migration = m }).ToArray()); + if (migration is not null) + { + var untilMigrationIdx = Array.IndexOf(migrations, migration); + if (untilMigrationIdx == -1) + throw new InvalidOperationException($"Migration {migration} not found"); + notAppliedMigrations = migrations[untilMigrationIdx..]; + await db.ExecuteAsync("CREATE TABLE IF NOT EXISTS \"__EFMigrationsHistory\" (\"MigrationId\" TEXT, \"ProductVersion\" TEXT)"); + await db.ExecuteAsync("INSERT INTO \"__EFMigrationsHistory\" VALUES (@migration, '8.0.0')", notAppliedMigrations.Select(m => new { migration = m }).ToArray()); + } await ctx.Database.MigrateAsync(); } diff --git a/BTCPayServer.Tests/DatabaseTests.cs b/BTCPayServer.Tests/DatabaseTests.cs index f213fd827d..60ee7cc586 100644 --- a/BTCPayServer.Tests/DatabaseTests.cs +++ b/BTCPayServer.Tests/DatabaseTests.cs @@ -3,7 +3,9 @@ using BTCPayServer.Payments; using Dapper; using Microsoft.EntityFrameworkCore; +using NBitcoin; using NBitcoin.Altcoins; +using NBitpayClient; using Newtonsoft.Json.Linq; using Xunit; using Xunit.Abstractions; @@ -18,6 +20,110 @@ public DatabaseTests(ITestOutputHelper helper):base(helper) { } + [Fact] + public async Task CanQueryMonitoredInvoices() + { + var tester = CreateDBTester(); + await tester.MigrateUntil(); + var invoiceRepository = tester.GetInvoiceRepository(); + using var ctx = tester.CreateContext(); + var conn = ctx.Database.GetDbConnection(); + + async Task AddPrompt(string invoiceId, string paymentMethodId, bool activated = true) + { + JObject prompt = new JObject(); + if (!activated) + prompt["inactive"] = true; + prompt["currency"] = "USD"; + var query = """ + UPDATE "Invoices" SET "Blob2" = jsonb_set('{"prompts": {}}'::JSONB || COALESCE("Blob2",'{}'), ARRAY['prompts','@paymentMethodId'], '@prompt'::JSONB) + WHERE "Id" = '@invoiceId' + """; + query = query.Replace("@paymentMethodId", paymentMethodId); + query = query.Replace("@prompt", prompt.ToString()); + query = query.Replace("@invoiceId", invoiceId); + Assert.Equal(1, await conn.ExecuteAsync(query)); + } + + await conn.ExecuteAsync(""" + INSERT INTO "Invoices" ("Id", "Created", "Status","Currency") VALUES + ('BTCOnly', NOW(), 'New', 'USD'), + ('LTCOnly', NOW(), 'New', 'USD'), + ('LTCAndBTC', NOW(), 'New', 'USD'), + ('LTCAndBTCLazy', NOW(), 'New', 'USD') + """); + foreach (var invoiceId in new string[] { "LTCOnly", "LTCAndBTCLazy", "LTCAndBTC" }) + { + await AddPrompt(invoiceId, "LTC-CHAIN", true); + } + foreach (var invoiceId in new string[] { "BTCOnly", "LTCAndBTC" }) + { + await AddPrompt(invoiceId, "BTC-CHAIN", true); + } + await AddPrompt("LTCAndBTCLazy", "BTC-CHAIN", false); + + var btc = PaymentMethodId.Parse("BTC-CHAIN"); + var ltc = PaymentMethodId.Parse("LTC-CHAIN"); + var invoices = await invoiceRepository.GetMonitoredInvoices(btc); + Assert.Equal(2, invoices.Length); + foreach (var invoiceId in new[] { "BTCOnly", "LTCAndBTC" }) + { + Assert.Contains(invoices, i => i.Id == invoiceId); + } + invoices = await invoiceRepository.GetMonitoredInvoices(btc, true); + Assert.Equal(3, invoices.Length); + foreach (var invoiceId in new[] { "BTCOnly", "LTCAndBTC", "LTCAndBTCLazy" }) + { + Assert.Contains(invoices, i => i.Id == invoiceId); + } + + invoices = await invoiceRepository.GetMonitoredInvoices(ltc); + Assert.Equal(3, invoices.Length); + foreach (var invoiceId in new[] { "LTCAndBTC", "LTCAndBTC", "LTCAndBTCLazy" }) + { + Assert.Contains(invoices, i => i.Id == invoiceId); + } + + await conn.ExecuteAsync(""" + INSERT INTO "Payments" ("Id", "InvoiceDataId", "PaymentMethodId", "Status", "Blob2", "Created", "Amount", "Currency") VALUES + ('1','LTCAndBTC', 'LTC-CHAIN', 'Processing', '{}'::JSONB, NOW(), 123, 'USD'), + ('2','LTCAndBTC', 'BTC-CHAIN', 'Processing', '{}'::JSONB, NOW(), 123, 'USD'), + ('3','LTCAndBTC', 'BTC-CHAIN', 'Processing', '{}'::JSONB, NOW(), 123, 'USD'), + ('4','LTCAndBTC', 'BTC-CHAIN', 'Settled', '{}'::JSONB, NOW(), 123, 'USD'); + + INSERT INTO "AddressInvoices" ("InvoiceDataId", "Address", "PaymentMethodId") VALUES + ('LTCAndBTC', 'BTC1', 'BTC-CHAIN'), + ('LTCAndBTC', 'BTC2', 'BTC-CHAIN'), + ('LTCAndBTC', 'LTC1', 'LTC-CHAIN'); + """); + + var invoice = Assert.Single(await invoiceRepository.GetMonitoredInvoices(ltc), i => i.Id == "LTCAndBTC"); + var payment = Assert.Single(invoice.GetPayments(false)); + Assert.Equal("1", payment.Id); + + foreach (var includeNonActivated in new[] { true, false }) + { + invoices = await invoiceRepository.GetMonitoredInvoices(btc, includeNonActivated); + invoice = Assert.Single(invoices, i => i.Id == "LTCAndBTC"); + var payments = invoice.GetPayments(false); + Assert.Equal(3, payments.Count); + + foreach (var paymentId in new[] { "2", "3", "4" }) + { + Assert.Contains(payments, p => p.Id == paymentId); + } + Assert.Equal(2, invoice.Addresses.Count); + foreach (var addr in new[] { "BTC1", "BTC2" }) + { + Assert.Contains(invoice.Addresses, p => p.Address == addr); + } + if (!includeNonActivated) + Assert.DoesNotContain(invoices, i => i.Id == "LTCAndBTCLazy"); + else + Assert.Contains(invoices, i => i.Id == "LTCAndBTCLazy"); + } + } + [Fact] public async Task CanMigrateInvoiceAddresses() { diff --git a/BTCPayServer/Data/InvoiceDataExtensions.cs b/BTCPayServer/Data/InvoiceDataExtensions.cs index 5fb5d9ecdc..7475a146e7 100644 --- a/BTCPayServer/Data/InvoiceDataExtensions.cs +++ b/BTCPayServer/Data/InvoiceDataExtensions.cs @@ -32,7 +32,7 @@ public static void SetBlob(this InvoiceData invoiceData, InvoiceEntity blob) #nullable enable public static PayoutMethodId? GetClosestPayoutMethodId(this InvoiceData invoice, IEnumerable pmids) { - var paymentMethodIds = invoice.Payments.Select(o => o.GetPaymentMethodId()).ToArray(); + var paymentMethodIds = invoice.Payments.Select(o => PaymentMethodId.Parse(o.PaymentMethodId)).ToArray(); if (paymentMethodIds.Length == 0) paymentMethodIds = invoice.GetBlob().GetPaymentPrompts().Select(p => p.PaymentMethodId).ToArray(); return PaymentMethodId.GetSimilarities(pmids, paymentMethodIds) diff --git a/BTCPayServer/Data/PaymentDataExtensions.cs b/BTCPayServer/Data/PaymentDataExtensions.cs index cf001cff56..11ca1033d8 100644 --- a/BTCPayServer/Data/PaymentDataExtensions.cs +++ b/BTCPayServer/Data/PaymentDataExtensions.cs @@ -38,16 +38,12 @@ public static PaymentData SetBlob(this PaymentData paymentData, PaymentMethodId paymentData.Blob2 = JToken.FromObject(blob, InvoiceDataExtensions.DefaultSerializer).ToString(Newtonsoft.Json.Formatting.None); return paymentData; } - public static PaymentMethodId GetPaymentMethodId(this PaymentData paymentData) - { - return PaymentMethodId.Parse(paymentData.PaymentMethodId); - } public static PaymentEntity GetBlob(this PaymentData paymentData) { var entity = JToken.Parse(paymentData.Blob2).ToObject(InvoiceDataExtensions.DefaultSerializer) ?? throw new FormatException($"Invalid {nameof(PaymentEntity)}"); entity.Status = paymentData.Status!.Value; entity.Currency = paymentData.Currency; - entity.PaymentMethodId = GetPaymentMethodId(paymentData); + entity.PaymentMethodId = PaymentMethodId.Parse(paymentData.PaymentMethodId); entity.Value = paymentData.Amount!.Value; entity.Id = paymentData.Id; entity.ReceivedTime = paymentData.Created!.Value; From f00a71922f5e0fbd45bebfd901c207c9fff9839c Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 24 Sep 2024 23:39:05 +0900 Subject: [PATCH 13/26] Optimize queries from payout processor at startup --- BTCPayServer.Tests/GreenfieldAPITests.cs | 7 ++++++- .../LightningAutomatedPayoutProcessor.cs | 7 +------ .../Services/Stores/StoreRepository.cs | 18 ++++++++++++++++++ BTCPayServer/Services/UserService.cs | 7 ------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index b7d791b41c..f651a60a6a 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -4122,7 +4122,12 @@ public async Task CanUseLNPayoutProcessor() var resp = await tester.CustomerLightningD.Pay(inv.BOLT11); Assert.Equal(PayResult.Ok, resp.Result); - + var store = tester.PayTester.GetService(); + Assert.True(await store.InternalNodePayoutAuthorized(admin.StoreId)); + Assert.False(await store.InternalNodePayoutAuthorized("blah")); + await admin.MakeAdmin(false); + Assert.False(await store.InternalNodePayoutAuthorized(admin.StoreId)); + await admin.MakeAdmin(true); var customerInvoice = await tester.CustomerLightningD.CreateInvoice(LightMoney.FromUnit(10, LightMoneyUnit.Satoshi), Guid.NewGuid().ToString(), TimeSpan.FromDays(40)); diff --git a/BTCPayServer/PayoutProcessors/Lightning/LightningAutomatedPayoutProcessor.cs b/BTCPayServer/PayoutProcessors/Lightning/LightningAutomatedPayoutProcessor.cs index 3cbe97a709..d604fb77c8 100644 --- a/BTCPayServer/PayoutProcessors/Lightning/LightningAutomatedPayoutProcessor.cs +++ b/BTCPayServer/PayoutProcessors/Lightning/LightningAutomatedPayoutProcessor.cs @@ -121,12 +121,7 @@ protected override async Task ProcessShouldSave(object paymentMethodConfig var processorBlob = GetBlob(PayoutProcessorSettings); var lightningSupportedPaymentMethod = (LightningPaymentMethodConfig)paymentMethodConfig; if (lightningSupportedPaymentMethod.IsInternalNode && - !(await Task.WhenAll((await _storeRepository.GetStoreUsers(PayoutProcessorSettings.StoreId)) - .Where(user => - user.StoreRole.ToPermissionSet(PayoutProcessorSettings.StoreId) - .Contains(Policies.CanModifyStoreSettings, PayoutProcessorSettings.StoreId)) - .Select(user => user.Id) - .Select(s => _userService.IsAdminUser(s)))).Any(b => b)) + !await _storeRepository.InternalNodePayoutAuthorized(PayoutProcessorSettings.StoreId)) { return false; } diff --git a/BTCPayServer/Services/Stores/StoreRepository.cs b/BTCPayServer/Services/Stores/StoreRepository.cs index 020bad5154..277dd1423c 100644 --- a/BTCPayServer/Services/Stores/StoreRepository.cs +++ b/BTCPayServer/Services/Stores/StoreRepository.cs @@ -9,6 +9,7 @@ using BTCPayServer.Data; using BTCPayServer.Events; using BTCPayServer.Migrations; +using Dapper; using Microsoft.EntityFrameworkCore; using NBitcoin; using NBitcoin.DataEncoders; @@ -637,6 +638,23 @@ private static bool IsDeadlock(DbUpdateException ex) { return ex.InnerException is Npgsql.PostgresException postgres && postgres.SqlState == "40P01"; } + + public async Task InternalNodePayoutAuthorized(string storeId) + { + using var ctx = _ContextFactory.CreateContext(); + return (await ctx.Database.GetDbConnection().ExecuteScalarAsync(""" + SELECT TRUE + FROM "UserStore" us + JOIN "StoreRoles" sr ON sr."Id" = us."Role" + JOIN "AspNetUserRoles" ur ON us."ApplicationUserId" = ur."UserId" + JOIN "AspNetRoles" r ON ur."RoleId" = r."Id" + WHERE + us."StoreDataId"=@storeId AND + r."NormalizedName"='SERVERADMIN' AND + 'btcpay.store.canmodifystoresettings' = ANY(sr."Permissions") + LIMIT 1; + """, new { storeId })) is true; + } } public record StoreRoleId diff --git a/BTCPayServer/Services/UserService.cs b/BTCPayServer/Services/UserService.cs index 9b8bffb0dd..1496c3fb51 100644 --- a/BTCPayServer/Services/UserService.cs +++ b/BTCPayServer/Services/UserService.cs @@ -160,13 +160,6 @@ public async Task SetUserApproval(string userId, bool approved, Uri reques return res.Succeeded; } - public async Task IsAdminUser(string userId) - { - using var scope = _serviceProvider.CreateScope(); - var userManager = scope.ServiceProvider.GetRequiredService>(); - return Roles.HasServerAdmin(await userManager.GetRolesAsync(new ApplicationUser() { Id = userId })); - } - public async Task IsAdminUser(ApplicationUser user) { using var scope = _serviceProvider.CreateScope(); From 747dacf3b134117686f2ad352f2b033de55a79a5 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Wed, 25 Sep 2024 18:23:10 +0900 Subject: [PATCH 14/26] Consolidate migrations from alpha (#6244) --- BTCPayServer.Data/BTCPayServer.Data.csproj | 4 -- .../DBScripts/004.MonitoredInvoices.sql | 17 +++---- .../DBScripts/005.PaymentsRenaming.sql | 17 ------- .../DBScripts/006.PaymentsRenaming.sql | 18 -------- .../DBScripts/007.PaymentsRenaming.sql | 18 -------- .../DBScripts/008.PaymentsRenaming.sql | 18 -------- .../20240827034505_migratepayouts.cs | 25 +++++++++++ .../Migrations/20240906010127_renamecol.cs | 45 ------------------- .../20240923065254_refactorpayments.cs | 1 - .../20240923071444_temprefactor2.cs | 31 ------------- ...cs => 20240924065254_monitoredinvoices.cs} | 2 +- .../20240924071444_temprefactor3.cs | 15 ------- .../20240924071444_temprefactor4.cs | 15 ------- .../20240924081444_temprefactor5.cs | 15 ------- BTCPayServer.Tests/TestUtils.cs | 2 +- 15 files changed, 36 insertions(+), 207 deletions(-) delete mode 100644 BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql delete mode 100644 BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql delete mode 100644 BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql delete mode 100644 BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql delete mode 100644 BTCPayServer.Data/Migrations/20240906010127_renamecol.cs delete mode 100644 BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs rename BTCPayServer.Data/Migrations/{20240919034505_monitoredinvoices.cs => 20240924065254_monitoredinvoices.cs} (87%) delete mode 100644 BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs delete mode 100644 BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs delete mode 100644 BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs diff --git a/BTCPayServer.Data/BTCPayServer.Data.csproj b/BTCPayServer.Data/BTCPayServer.Data.csproj index 7e1dbd5f8a..0a3c73e521 100644 --- a/BTCPayServer.Data/BTCPayServer.Data.csproj +++ b/BTCPayServer.Data/BTCPayServer.Data.csproj @@ -22,9 +22,5 @@ - - - - diff --git a/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql b/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql index b89f396ffc..6736ff8838 100644 --- a/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql +++ b/BTCPayServer.Data/DBScripts/004.MonitoredInvoices.sql @@ -4,19 +4,20 @@ RETURNS JSONB AS $$ $$ LANGUAGE sql IMMUTABLE; -CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT) -RETURNS TABLE (invoice_id TEXT, payment_id TEXT) AS $$ +CREATE OR REPLACE FUNCTION get_monitored_invoices(arg_payment_method_id TEXT, include_non_activated BOOLEAN) +RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ WITH cte AS ( -- Get all the invoices which are pending. Even if no payments. -SELECT i."Id" invoice_id, p."Id" payment_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" WHERE is_pending(i."Status") UNION ALL -- For invoices not pending, take all of those which have pending payments -SELECT i."Id", p."Id" FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" +SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) SELECT cte.* FROM cte -LEFT JOIN "Payments" p ON cte.payment_id=p."Id" -LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" -WHERE (p."Type" IS NOT NULL AND p."Type" = payment_method_id) OR - (p."Type" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE); +JOIN "Invoices" i ON cte.invoice_id=i."Id" +LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_method_id=p."PaymentMethodId" +WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = arg_payment_method_id) OR + (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", arg_payment_method_id) IS NOT NULL AND + (include_non_activated IS TRUE OR (get_prompt(i."Blob2", arg_payment_method_id)->'inactive')::BOOLEAN IS NOT TRUE)); $$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql deleted file mode 100644 index b52c5e7420..0000000000 --- a/BTCPayServer.Data/DBScripts/005.PaymentsRenaming.sql +++ /dev/null @@ -1,17 +0,0 @@ -DROP FUNCTION get_monitored_invoices; -CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT) -RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ -WITH cte AS ( --- Get all the invoices which are pending. Even if no payments. -SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(i."Status") -UNION ALL --- For invoices not pending, take all of those which have pending payments -SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) -SELECT cte.* FROM cte -LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" -LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" -WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = payment_method_id) OR - (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE); -$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql deleted file mode 100644 index 393b411800..0000000000 --- a/BTCPayServer.Data/DBScripts/006.PaymentsRenaming.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP FUNCTION get_monitored_invoices; -CREATE OR REPLACE FUNCTION get_monitored_invoices(payment_method_id TEXT, include_non_activated BOOLEAN) -RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ -WITH cte AS ( --- Get all the invoices which are pending. Even if no payments. -SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(i."Status") -UNION ALL --- For invoices not pending, take all of those which have pending payments -SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) -SELECT cte.* FROM cte -LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" -LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" -WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = payment_method_id) OR - (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", payment_method_id) IS NOT NULL AND - (include_non_activated IS TRUE OR (get_prompt(i."Blob2", payment_method_id)->'activated')::BOOLEAN IS NOT FALSE)); -$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql deleted file mode 100644 index ae31a98536..0000000000 --- a/BTCPayServer.Data/DBScripts/007.PaymentsRenaming.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP FUNCTION get_monitored_invoices; -CREATE OR REPLACE FUNCTION get_monitored_invoices(arg_payment_method_id TEXT, include_non_activated BOOLEAN) -RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ -WITH cte AS ( --- Get all the invoices which are pending. Even if no payments. -SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(i."Status") -UNION ALL --- For invoices not pending, take all of those which have pending payments -SELECT i."Id", p."Id", p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) -SELECT cte.* FROM cte -LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_id=p."PaymentMethodId" -LEFT JOIN "Invoices" i ON cte.invoice_id=i."Id" -WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = arg_payment_method_id) OR - (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", arg_payment_method_id) IS NOT NULL AND - (include_non_activated IS TRUE OR (get_prompt(i."Blob2", arg_payment_method_id)->'activated')::BOOLEAN IS NOT FALSE)); -$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql b/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql deleted file mode 100644 index c6e4439710..0000000000 --- a/BTCPayServer.Data/DBScripts/008.PaymentsRenaming.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP FUNCTION get_monitored_invoices; -CREATE OR REPLACE FUNCTION get_monitored_invoices(arg_payment_method_id TEXT, include_non_activated BOOLEAN) -RETURNS TABLE (invoice_id TEXT, payment_id TEXT, payment_method_id TEXT) AS $$ -WITH cte AS ( --- Get all the invoices which are pending. Even if no payments. -SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i LEFT JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(i."Status") -UNION ALL --- For invoices not pending, take all of those which have pending payments -SELECT i."Id" invoice_id, p."Id" payment_id, p."PaymentMethodId" payment_method_id FROM "Invoices" i INNER JOIN "Payments" p ON i."Id" = p."InvoiceDataId" - WHERE is_pending(p."Status") AND NOT is_pending(i."Status")) -SELECT cte.* FROM cte -JOIN "Invoices" i ON cte.invoice_id=i."Id" -LEFT JOIN "Payments" p ON cte.payment_id=p."Id" AND cte.payment_method_id=p."PaymentMethodId" -WHERE (p."PaymentMethodId" IS NOT NULL AND p."PaymentMethodId" = arg_payment_method_id) OR - (p."PaymentMethodId" IS NULL AND get_prompt(i."Blob2", arg_payment_method_id) IS NOT NULL AND - (include_non_activated IS TRUE OR (get_prompt(i."Blob2", arg_payment_method_id)->'inactive')::BOOLEAN IS NOT TRUE)); -$$ LANGUAGE SQL STABLE; diff --git a/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs b/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs index f1aea1fe17..3829714320 100644 --- a/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs +++ b/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs @@ -11,5 +11,30 @@ namespace BTCPayServer.Migrations [DBScript("002.RefactorPayouts.sql")] public partial class migratepayouts : DBScriptsMigration { + protected override void Up(MigrationBuilder migrationBuilder) + { + base.Up(migrationBuilder); + migrationBuilder.RenameColumn( + name: "Destination", + table: "Payouts", + newName: "DedupId"); + migrationBuilder.RenameIndex( + name: "IX_Payouts_Destination_State", + table: "Payouts", + newName: "IX_Payouts_DedupId_State"); + migrationBuilder.RenameColumn( + name: "PaymentMethod", + table: "PayoutProcessors", + newName: "PayoutMethodId"); + + migrationBuilder.Sql(""" + UPDATE "PayoutProcessors" + SET + "PayoutMethodId" = CASE WHEN STRPOS("PayoutMethodId", '_') = 0 THEN "PayoutMethodId" || '-CHAIN' + WHEN STRPOS("PayoutMethodId", '_LightningLike') = 0 THEN "PayoutMethodId" || '-LN' + WHEN STRPOS("PayoutMethodId", '_LNURLPAY') = 0 THEN "PayoutMethodId" || '-LN' + ELSE "PayoutMethodId" END; + """); + } } } diff --git a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs b/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs deleted file mode 100644 index 928b24310d..0000000000 --- a/BTCPayServer.Data/Migrations/20240906010127_renamecol.cs +++ /dev/null @@ -1,45 +0,0 @@ -using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace BTCPayServer.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20240906010127_renamecol")] - public partial class renamecol : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.RenameColumn( - name: "Destination", - table: "Payouts", - newName: "DedupId"); - - migrationBuilder.RenameIndex( - name: "IX_Payouts_Destination_State", - table: "Payouts", - newName: "IX_Payouts_DedupId_State"); - migrationBuilder.RenameColumn( - name: "PaymentMethod", - table: "PayoutProcessors", - newName: "PayoutMethodId"); - - migrationBuilder.Sql(""" - UPDATE "PayoutProcessors" - SET - "PayoutMethodId" = CASE WHEN STRPOS("PayoutMethodId", '_') = 0 THEN "PayoutMethodId" || '-CHAIN' - WHEN STRPOS("PayoutMethodId", '_LightningLike') = 0 THEN "PayoutMethodId" || '-LN' - WHEN STRPOS("PayoutMethodId", '_LNURLPAY') = 0 THEN "PayoutMethodId" || '-LN' - ELSE "PayoutMethodId" END; - """); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - } - } -} diff --git a/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs index 80ad50b00d..1c9d3976cd 100644 --- a/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs +++ b/BTCPayServer.Data/Migrations/20240923065254_refactorpayments.cs @@ -9,7 +9,6 @@ namespace BTCPayServer.Migrations [DbContext(typeof(ApplicationDbContext))] [Migration("20240923065254_refactorpayments")] - [DBScript("005.PaymentsRenaming.sql")] public partial class refactorpayments : DBScriptsMigration { /// diff --git a/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs b/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs deleted file mode 100644 index 6b34066222..0000000000 --- a/BTCPayServer.Data/Migrations/20240923071444_temprefactor2.cs +++ /dev/null @@ -1,31 +0,0 @@ -using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace BTCPayServer.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20240923071444_temprefactor2")] - public partial class temprefactor2 : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropPrimaryKey( - name: "PK_AddressInvoices", - table: "AddressInvoices"); - - migrationBuilder.AddPrimaryKey( - name: "PK_AddressInvoices", - table: "AddressInvoices", - columns: new[] { "Address", "PaymentMethodId" }); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - } - } -} diff --git a/BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs b/BTCPayServer.Data/Migrations/20240924065254_monitoredinvoices.cs similarity index 87% rename from BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs rename to BTCPayServer.Data/Migrations/20240924065254_monitoredinvoices.cs index 199b9070c7..2dbbff68f1 100644 --- a/BTCPayServer.Data/Migrations/20240919034505_monitoredinvoices.cs +++ b/BTCPayServer.Data/Migrations/20240924065254_monitoredinvoices.cs @@ -7,7 +7,7 @@ namespace BTCPayServer.Migrations { [DbContext(typeof(ApplicationDbContext))] - [Migration("20240919034505_monitoredinvoices")] + [Migration("20240924065254_monitoredinvoices")] [DBScript("004.MonitoredInvoices.sql")] public partial class monitoredinvoices : DBScriptsMigration { diff --git a/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs b/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs deleted file mode 100644 index 6884ded7d2..0000000000 --- a/BTCPayServer.Data/Migrations/20240924071444_temprefactor3.cs +++ /dev/null @@ -1,15 +0,0 @@ -using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace BTCPayServer.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20240924071444_temprefactor3")] - [DBScript("006.PaymentsRenaming.sql")] - public partial class temprefactor3 : DBScriptsMigration - { - } -} diff --git a/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs b/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs deleted file mode 100644 index 968a5c6ced..0000000000 --- a/BTCPayServer.Data/Migrations/20240924071444_temprefactor4.cs +++ /dev/null @@ -1,15 +0,0 @@ -using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace BTCPayServer.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20240924071444_temprefactor4")] - [DBScript("007.PaymentsRenaming.sql")] - public partial class temprefactor4 : DBScriptsMigration - { - } -} diff --git a/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs b/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs deleted file mode 100644 index f6efaab879..0000000000 --- a/BTCPayServer.Data/Migrations/20240924081444_temprefactor5.cs +++ /dev/null @@ -1,15 +0,0 @@ -using BTCPayServer.Data; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace BTCPayServer.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20240924081444_temprefactor5")] - [DBScript("008.PaymentsRenaming.sql")] - public partial class temprefactor5 : DBScriptsMigration - { - } -} diff --git a/BTCPayServer.Tests/TestUtils.cs b/BTCPayServer.Tests/TestUtils.cs index 1aebb0ae87..09c3b0cc37 100644 --- a/BTCPayServer.Tests/TestUtils.cs +++ b/BTCPayServer.Tests/TestUtils.cs @@ -16,7 +16,7 @@ namespace BTCPayServer.Tests public static class TestUtils { #if DEBUG && !SHORT_TIMEOUT - public const int TestTimeout = 20_000; + public const int TestTimeout = 600_000; #else public const int TestTimeout = 90_000; #endif From e16b4062b55bfe9e6d4baa3b62d5d5c178d06aae Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 25 Sep 2024 18:50:49 +0900 Subject: [PATCH 15/26] Fix payout processor migration --- .../Migrations/20240827034505_migratepayouts.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs b/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs index 3829714320..fb6ec1d2bd 100644 --- a/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs +++ b/BTCPayServer.Data/Migrations/20240827034505_migratepayouts.cs @@ -31,9 +31,9 @@ protected override void Up(MigrationBuilder migrationBuilder) UPDATE "PayoutProcessors" SET "PayoutMethodId" = CASE WHEN STRPOS("PayoutMethodId", '_') = 0 THEN "PayoutMethodId" || '-CHAIN' - WHEN STRPOS("PayoutMethodId", '_LightningLike') = 0 THEN "PayoutMethodId" || '-LN' - WHEN STRPOS("PayoutMethodId", '_LNURLPAY') = 0 THEN "PayoutMethodId" || '-LN' - ELSE "PayoutMethodId" END; + WHEN STRPOS("PayoutMethodId", '_LightningLike') > 0 THEN split_part("PayoutMethodId", '_LightningLike', 1) || '-LN' + WHEN STRPOS("PayoutMethodId", '_LNURLPAY') > 0 THEN split_part("PayoutMethodId",'_LNURLPAY', 1) || '-LN' + ELSE "PayoutMethodId" END """); } } From 336f2d88e95ed5a8bded7eec869bdc91f3ff1cc7 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 25 Sep 2024 21:53:15 +0900 Subject: [PATCH 16/26] Fix flaky CanManageWallet Clicking on Sign Transaction in the Wallet Send page, will, when a hot wallet is setup, use PostRedirect page to redirect to the broadcast screen. The problem was that sometimes, s.Driver.PageSource would return this PostRedirect page rather than the broadcast page. Waiting for an element of the broadcast page fixes this issue. --- BTCPayServer.Tests/SeleniumTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/BTCPayServer.Tests/SeleniumTests.cs b/BTCPayServer.Tests/SeleniumTests.cs index ea94e2603c..15fa473a8d 100644 --- a/BTCPayServer.Tests/SeleniumTests.cs +++ b/BTCPayServer.Tests/SeleniumTests.cs @@ -1973,6 +1973,7 @@ await s.Server.ExplorerNode.SendToAddressAsync(BitcoinAddress.Create(receiveAddr var jack = new Key().PubKey.Hash.GetAddress(Network.RegTest); SetTransactionOutput(s, 0, jack, 0.01m); s.Driver.FindElement(By.Id("SignTransaction")).Click(); + s.Driver.WaitForElement(By.CssSelector("button[value=broadcast]")); Assert.Contains(jack.ToString(), s.Driver.PageSource); Assert.Contains("0.01000000", s.Driver.PageSource); Assert.EndsWith("psbt/ready", s.Driver.Url); From 056f850268196d1b8524761e9d2bad35c0f41946 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Wed, 25 Sep 2024 23:10:13 +0900 Subject: [PATCH 17/26] Optimize load time of StoreRoles related pages/routes (#6245) --- .../GreenfieldServerRolesController.cs | 2 +- .../GreenfieldStoreRolesController.cs | 2 +- .../Controllers/UIServerController.Roles.cs | 2 +- .../Controllers/UIStoresController.Roles.cs | 2 +- .../Services/Stores/StoreRepository.cs | 21 ++++++++++++------- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldServerRolesController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldServerRolesController.cs index 3db6c1dfa2..7448b76a49 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldServerRolesController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldServerRolesController.cs @@ -28,7 +28,7 @@ public GreenfieldServerRolesController(StoreRepository storeRepository) [HttpGet("~/api/v1/server/roles")] public async Task GetServerRoles() { - return Ok(FromModel(await _storeRepository.GetStoreRoles(null, false, false))); + return Ok(FromModel(await _storeRepository.GetStoreRoles(null, false))); } private List FromModel(StoreRepository.StoreRole[] data) { diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStoreRolesController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStoreRolesController.cs index 5c69198665..14621f5fee 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStoreRolesController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStoreRolesController.cs @@ -31,7 +31,7 @@ public async Task GetStoreRoles(string storeId) var store = HttpContext.GetStoreData(); return store == null ? StoreNotFound() - : Ok(FromModel(await _storeRepository.GetStoreRoles(storeId, false, false))); + : Ok(FromModel(await _storeRepository.GetStoreRoles(storeId, false))); } private List FromModel(StoreRepository.StoreRole[] data) diff --git a/BTCPayServer/Controllers/UIServerController.Roles.cs b/BTCPayServer/Controllers/UIServerController.Roles.cs index e4a2ea2546..1e9f40405b 100644 --- a/BTCPayServer/Controllers/UIServerController.Roles.cs +++ b/BTCPayServer/Controllers/UIServerController.Roles.cs @@ -19,7 +19,7 @@ public async Task ListRoles( string sortOrder = null ) { - var roles = await _StoreRepository.GetStoreRoles(null, true); + var roles = await _StoreRepository.GetStoreRoles(null); var defaultRole = (await _StoreRepository.GetDefaultRole()).Role; model ??= new RolesViewModel(); model.DefaultRole = defaultRole; diff --git a/BTCPayServer/Controllers/UIStoresController.Roles.cs b/BTCPayServer/Controllers/UIStoresController.Roles.cs index b5bc9c34a5..ea7a34e72f 100644 --- a/BTCPayServer/Controllers/UIStoresController.Roles.cs +++ b/BTCPayServer/Controllers/UIStoresController.Roles.cs @@ -21,7 +21,7 @@ public async Task ListRoles( string sortOrder = null ) { - var roles = await storeRepository.GetStoreRoles(storeId, true); + var roles = await storeRepository.GetStoreRoles(storeId); var defaultRole = (await storeRepository.GetDefaultRole()).Role; model ??= new RolesViewModel(); model.DefaultRole = defaultRole; diff --git a/BTCPayServer/Services/Stores/StoreRepository.cs b/BTCPayServer/Services/Stores/StoreRepository.cs index 277dd1423c..a6ecb3311e 100644 --- a/BTCPayServer/Services/Stores/StoreRepository.cs +++ b/BTCPayServer/Services/Stores/StoreRepository.cs @@ -14,6 +14,7 @@ using NBitcoin; using NBitcoin.DataEncoders; using Newtonsoft.Json; +using static BTCPayServer.Services.Stores.StoreRepository; namespace BTCPayServer.Services.Stores { @@ -81,14 +82,20 @@ public PermissionSet ToPermissionSet(string storeId) public bool? IsUsed { get; set; } } #nullable enable - public async Task GetStoreRoles(string? storeId, bool includeUsers = false, bool storeOnly = false) + public async Task GetStoreRoles(string? storeId, bool storeOnly = false) { await using var ctx = _ContextFactory.CreateContext(); - var query = ctx.StoreRoles.Where(u => (storeOnly && u.StoreDataId == storeId) || (!storeOnly && (u.StoreDataId == null || u.StoreDataId == storeId))); - if (includeUsers) - { - query = query.Include(u => u.Users); - } + var query = ctx.StoreRoles + .Where(u => (storeOnly && u.StoreDataId == storeId) || (!storeOnly && (u.StoreDataId == null || u.StoreDataId == storeId))) + // Not calling ToStoreRole here because we don't want to load users in the DB query + .Select(u => new StoreRole() + { + Id = u.Id, + Role = u.Role, + Permissions = u.Permissions, + IsServerRole = u.StoreDataId == null, + IsUsed = u.Users.Any() + }); var roles = await query.ToArrayAsync(); // return ordered: default role comes first, then server-wide roles in specified order, followed by store roles @@ -99,7 +106,7 @@ public async Task GetStoreRoles(string? storeId, bool includeUsers if (role.Role == defaultRole.Role) return -1; int index = Array.IndexOf(defaultOrder, role.Role); return index == -1 ? int.MaxValue : index; - }).Select(ToStoreRole).ToArray(); + }).ToArray(); } public async Task GetDefaultRole() From 90635ffc4e378c9581b3b4d5d63f638946ade230 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 25 Sep 2024 23:11:40 +0900 Subject: [PATCH 18/26] Remove BTCPAY_EXPERIMENTALV2_CONFIRM --- BTCPayServer.Tests/docker-compose.altcoins.yml | 1 - BTCPayServer.Tests/docker-compose.yml | 1 - BTCPayServer/Program.cs | 7 ------- BTCPayServer/Properties/launchSettings.json | 4 ---- 4 files changed, 13 deletions(-) diff --git a/BTCPayServer.Tests/docker-compose.altcoins.yml b/BTCPayServer.Tests/docker-compose.altcoins.yml index cc51232389..ef4df5405b 100644 --- a/BTCPayServer.Tests/docker-compose.altcoins.yml +++ b/BTCPayServer.Tests/docker-compose.altcoins.yml @@ -12,7 +12,6 @@ services: args: CONFIGURATION_NAME: Release environment: - TESTS_EXPERIMENTALV2_CONFIRM: "true" TESTS_BTCRPCCONNECTION: server=http://bitcoind:43782;ceiwHEbqWI83:DwubwWsoo3 TESTS_LTCRPCCONNECTION: server=http://litecoind:43782;ceiwHEbqWI83:DwubwWsoo3 TESTS_BTCNBXPLORERURL: http://nbxplorer:32838/ diff --git a/BTCPayServer.Tests/docker-compose.yml b/BTCPayServer.Tests/docker-compose.yml index a82b8af916..de52b507e2 100644 --- a/BTCPayServer.Tests/docker-compose.yml +++ b/BTCPayServer.Tests/docker-compose.yml @@ -12,7 +12,6 @@ services: args: CONFIGURATION_NAME: Release environment: - TESTS_EXPERIMENTALV2_CONFIRM: "true" TESTS_BTCRPCCONNECTION: server=http://bitcoind:43782;ceiwHEbqWI83:DwubwWsoo3 TESTS_BTCNBXPLORERURL: http://nbxplorer:32838/ TESTS_POSTGRES: User ID=postgres;Include Error Detail=true;Host=postgres;Port=5432;Database=btcpayserver diff --git a/BTCPayServer/Program.cs b/BTCPayServer/Program.cs index 5969d8c2e1..a333ea35b5 100644 --- a/BTCPayServer/Program.cs +++ b/BTCPayServer/Program.cs @@ -44,13 +44,6 @@ static async Task Main(string[] args) confBuilder.AddJsonFile("appsettings.dev.json", true, false); #endif conf = confBuilder.Build(); - - - var confirm = conf.GetOrDefault("EXPERIMENTALV2_CONFIRM", false); - if(!confirm) - { - throw new ConfigException("You are running an experimental version of BTCPay Server that is the basis for v2. Many things will change and break, including irreversible database migrations. THERE IS NO WAY BACK. Please confirm you understand this by setting the setting EXPERIMENTALV2_CONFIRM=true"); - } var builder = new WebHostBuilder() .UseKestrel() .UseConfiguration(conf) diff --git a/BTCPayServer/Properties/launchSettings.json b/BTCPayServer/Properties/launchSettings.json index c10ddd4b4b..8f591de7e0 100644 --- a/BTCPayServer/Properties/launchSettings.json +++ b/BTCPayServer/Properties/launchSettings.json @@ -4,7 +4,6 @@ "commandName": "Project", "launchBrowser": true, "environmentVariables": { - "BTCPAY_EXPERIMENTALV2_CONFIRM": "true", "BTCPAY_NETWORK": "regtest", "BTCPAY_LAUNCHSETTINGS": "true", "BTCPAY_BTCLIGHTNING": "type=clightning;server=tcp://127.0.0.1:30993/", @@ -38,7 +37,6 @@ "commandName": "Project", "launchBrowser": true, "environmentVariables": { - "BTCPAY_EXPERIMENTALV2_CONFIRM": "true", "BTCPAY_NETWORK": "regtest", "BTCPAY_LAUNCHSETTINGS": "true", "BTCPAY_PORT": "14142", @@ -76,7 +74,6 @@ "commandName": "Project", "launchBrowser": true, "environmentVariables": { - "BTCPAY_EXPERIMENTALV2_CONFIRM": "true", "BTCPAY_NETWORK": "regtest", "BTCPAY_LAUNCHSETTINGS": "true", "BTCPAY_PORT": "14142", @@ -117,7 +114,6 @@ "commandName": "Project", "launchBrowser": true, "environmentVariables": { - "BTCPAY_EXPERIMENTALV2_CONFIRM": "true", "BTCPAY_NETWORK": "regtest", "BTCPAY_LAUNCHSETTINGS": "true", "BTCPAY_PORT": "14142", From 363b60385b05a94d61a94ff753e7b1db28b55bff Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Thu, 26 Sep 2024 11:25:45 +0900 Subject: [PATCH 19/26] Renaming various properties in the Payouts API (#6246) * Rename Payouts Currency/OriginalCurrency * Rename Payout Processor PayoutMethodIds * Rename paymentMethods to payoutMethodIds * Rename payoutMethodIds to payoutMethods --- .../Models/CreatePullPaymentRequest.cs | 2 +- BTCPayServer.Client/Models/PayoutData.cs | 7 +-- .../Models/PayoutProcessorData.cs | 2 +- BTCPayServer.Tests/GreenfieldAPITests.cs | 44 +++++++++---------- BTCPayServer.Tests/UnitTest1.cs | 4 +- .../GreenField/GreenfieldInvoiceController.cs | 2 +- .../GreenfieldPayoutProcessorsController.cs | 2 +- .../GreenfieldPullPaymentController.cs | 19 ++++---- ...atedLightningPayoutProcessorsController.cs | 4 +- ...omatedOnChainPayoutProcessorsController.cs | 4 +- ...eenfieldStorePayoutProcessorsController.cs | 4 +- .../Controllers/UIInvoiceController.UI.cs | 2 +- ...torePullPaymentsController.PullPayments.cs | 4 +- .../PullPaymentHostedService.cs | 6 +-- .../WalletViewModels/PullPaymentsModel.cs | 2 - ...ningAutomatedPayoutProcessorsController.cs | 4 +- ...hainAutomatedPayoutProcessorsController.cs | 4 +- .../PayoutProcessorService.cs | 8 ++-- .../swagger.template.payout-processors.json | 2 +- .../v1/swagger.template.pull-payments.json | 26 +++++++++-- 20 files changed, 85 insertions(+), 67 deletions(-) diff --git a/BTCPayServer.Client/Models/CreatePullPaymentRequest.cs b/BTCPayServer.Client/Models/CreatePullPaymentRequest.cs index 2d4a753063..ec6ae7a60a 100644 --- a/BTCPayServer.Client/Models/CreatePullPaymentRequest.cs +++ b/BTCPayServer.Client/Models/CreatePullPaymentRequest.cs @@ -19,7 +19,7 @@ public class CreatePullPaymentRequest public DateTimeOffset? ExpiresAt { get; set; } [JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))] public DateTimeOffset? StartsAt { get; set; } - public string[] PaymentMethods { get; set; } + public string[] PayoutMethods { get; set; } public bool AutoApproveClaims { get; set; } } } diff --git a/BTCPayServer.Client/Models/PayoutData.cs b/BTCPayServer.Client/Models/PayoutData.cs index 44077afa47..aec6373697 100644 --- a/BTCPayServer.Client/Models/PayoutData.cs +++ b/BTCPayServer.Client/Models/PayoutData.cs @@ -22,11 +22,12 @@ public class PayoutData public string PullPaymentId { get; set; } public string Destination { get; set; } public string PayoutMethodId { get; set; } - public string CryptoCode { get; set; } [JsonConverter(typeof(NumericStringJsonConverter))] - public decimal Amount { get; set; } + public decimal OriginalAmount { get; set; } + public string OriginalCurrency { get; set; } + public string PayoutCurrency { get; set; } [JsonConverter(typeof(NumericStringJsonConverter))] - public decimal? PaymentMethodAmount { get; set; } + public decimal? PayoutAmount { get; set; } [JsonConverter(typeof(StringEnumConverter))] public PayoutState State { get; set; } public int Revision { get; set; } diff --git a/BTCPayServer.Client/Models/PayoutProcessorData.cs b/BTCPayServer.Client/Models/PayoutProcessorData.cs index b8eeb3a487..95500560e3 100644 --- a/BTCPayServer.Client/Models/PayoutProcessorData.cs +++ b/BTCPayServer.Client/Models/PayoutProcessorData.cs @@ -4,6 +4,6 @@ public class PayoutProcessorData { public string Name { get; set; } public string FriendlyName { get; set; } - public string[] PaymentMethods { get; set; } + public string[] PayoutMethods { get; set; } } } diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index f651a60a6a..8eb6fdf802 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -1104,7 +1104,7 @@ public async Task CanUsePullPaymentViaAPI() Description = "Test description", Amount = 12.3m, Currency = "BTC", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); void VerifyResult() @@ -1135,7 +1135,7 @@ void VerifyResult() Name = "Test 2", Amount = 12.3m, Currency = "BTC", - PaymentMethods = new[] { "BTC" }, + PayoutMethods = new[] { "BTC" }, BOLT11Expiration = TimeSpan.FromDays(31.0) }); Assert.Equal(TimeSpan.FromDays(31.0), test2.BOLT11Expiration); @@ -1182,13 +1182,13 @@ void VerifyResult() payouts = await unauthenticated.GetPayouts(pps[0].Id); var payout2 = Assert.Single(payouts); - Assert.Equal(payout.Amount, payout2.Amount); + Assert.Equal(payout.OriginalAmount, payout2.OriginalAmount); Assert.Equal(payout.Id, payout2.Id); Assert.Equal(destination, payout2.Destination); Assert.Equal(PayoutState.AwaitingApproval, payout.State); Assert.Equal("BTC-CHAIN", payout2.PayoutMethodId); - Assert.Equal("BTC", payout2.CryptoCode); - Assert.Null(payout.PaymentMethodAmount); + Assert.Equal("BTC", payout2.PayoutCurrency); + Assert.Null(payout.PayoutAmount); TestLogs.LogInformation("Can't overdraft"); @@ -1230,7 +1230,7 @@ void VerifyResult() Amount = 12.3m, StartsAt = start, Currency = "BTC", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); Assert.Equal(start, inFuture.StartsAt); Assert.Null(inFuture.ExpiresAt); @@ -1248,7 +1248,7 @@ void VerifyResult() Amount = 12.3m, ExpiresAt = expires, Currency = "BTC", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); await this.AssertAPIError("expired", async () => await unauthenticated.CreatePayout(inPast.Id, new CreatePayoutRequest() { @@ -1272,7 +1272,7 @@ void VerifyResult() Name = "Test USD", Amount = 5000m, Currency = "USD", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); await this.AssertAPIError("lnurl-not-supported", async () => await unauthenticated.GetPullPaymentLNURL(pp.Id)); @@ -1297,8 +1297,8 @@ void VerifyResult() Revision = payout.Revision }); Assert.Equal(PayoutState.AwaitingPayment, payout.State); - Assert.NotNull(payout.PaymentMethodAmount); - Assert.Equal(1.0m, payout.PaymentMethodAmount); // 1 BTC == 5000 USD in tests + Assert.NotNull(payout.PayoutAmount); + Assert.Equal(1.0m, payout.PayoutAmount); // 1 BTC == 5000 USD in tests await this.AssertAPIError("invalid-state", async () => await client.ApprovePayout(storeId, payout.Id, new ApprovePayoutRequest() { Revision = payout.Revision @@ -1310,7 +1310,7 @@ void VerifyResult() Name = "Test 2", Amount = 12.303228134m, Currency = "BTC", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); destination = (await tester.ExplorerNode.GetNewAddressAsync()).ToString(); payout = await unauthenticated.CreatePayout(test3.Id, new CreatePayoutRequest() @@ -1320,8 +1320,8 @@ void VerifyResult() }); payout = await client.ApprovePayout(storeId, payout.Id, new ApprovePayoutRequest()); // The payout should round the value of the payment down to the network of the payment method - Assert.Equal(12.30322814m, payout.PaymentMethodAmount); - Assert.Equal(12.303228134m, payout.Amount); + Assert.Equal(12.30322814m, payout.PayoutAmount); + Assert.Equal(12.303228134m, payout.OriginalAmount); await client.MarkPayoutPaid(storeId, payout.Id); payout = (await client.GetPayouts(payout.PullPaymentId)).First(data => data.Id == payout.Id); @@ -1334,7 +1334,7 @@ void VerifyResult() Name = "Test 3", Amount = 12.303228134m, Currency = "BTC", - PaymentMethods = new[] { "BTC", "BTC-LightningNetwork", "BTC_LightningLike" } + PayoutMethods = new[] { "BTC", "BTC-LightningNetwork", "BTC_LightningLike" } }); var lnrURLs = await unauthenticated.GetPullPaymentLNURL(test4.Id); Assert.IsType(lnrURLs.LNURLBech32); @@ -1409,7 +1409,7 @@ void VerifyResult() Name = "Test SATS", Amount = 21000, Currency = "SATS", - PaymentMethods = new[] { "BTC", "BTC-LightningNetwork", "BTC_LightningLike" } + PayoutMethods = new[] { "BTC", "BTC-LightningNetwork", "BTC_LightningLike" } }); lnrURLs = await unauthenticated.GetPullPaymentLNURL(testSats.Id); Assert.IsType(lnrURLs.LNURLBech32); @@ -1427,7 +1427,7 @@ await AssertPermissionError(Policies.CanCreatePullPayments, async () => Amount = 100, Currency = "USD", Name = "pull payment", - PaymentMethods = new[] { "BTC" }, + PayoutMethods = new[] { "BTC" }, AutoApproveClaims = true }); }); @@ -1447,7 +1447,7 @@ await AssertPermissionError(Policies.CanCreatePullPayments, async () => Amount = 100, Currency = "USD", Name = "pull payment", - PaymentMethods = new[] { "BTC" }, + PayoutMethods = new[] { "BTC" }, AutoApproveClaims = true }); @@ -4188,7 +4188,7 @@ await TestUtils.EventuallyAsync(async () => PayoutMethodId = "BTC_LightningNetwork", Destination = customerInvoice.BOLT11 }); - Assert.Equal(payout2.Amount, new Money(100, MoneyUnit.Satoshi).ToDecimal(MoneyUnit.BTC)); + Assert.Equal(payout2.OriginalAmount, new Money(100, MoneyUnit.Satoshi).ToDecimal(MoneyUnit.BTC)); } [Fact(Timeout = 60 * 2 * 1000)] @@ -4232,7 +4232,7 @@ public async Task CanUsePayoutProcessorsThroughAPI() Amount = 100, Currency = "USD", Name = "pull payment", - PaymentMethods = new[] { "BTC" } + PayoutMethods = new[] { "BTC" } }); var notapprovedPayoutWithPullPayment = await adminClient.CreatePayout(admin.StoreId, new CreatePayoutThroughStoreRequest() @@ -4258,7 +4258,7 @@ await adminClient.ApprovePayout(admin.StoreId, notApprovedPayoutWithoutPullPayme Assert.Equal(3, payouts.Length); Assert.Empty(payouts.Where(data => data.State == PayoutState.AwaitingApproval)); - Assert.Empty(payouts.Where(data => data.PaymentMethodAmount is null)); + Assert.Empty(payouts.Where(data => data.PayoutAmount is null)); Assert.Empty(await adminClient.ShowOnChainWalletTransactions(admin.StoreId, "BTC")); @@ -4271,12 +4271,12 @@ await adminClient.UpdateStoreOnChainAutomatedPayoutProcessors(admin.StoreId, "BT Assert.Equal(3600, Assert.Single(await adminClient.GetStoreOnChainAutomatedPayoutProcessors(admin.StoreId, "BTC")).IntervalSeconds.TotalSeconds); var tpGen = Assert.Single(await adminClient.GetPayoutProcessors(admin.StoreId)); - Assert.Equal("BTC-CHAIN", Assert.Single(tpGen.PaymentMethods)); + Assert.Equal("BTC-CHAIN", Assert.Single(tpGen.PayoutMethods)); //still too poor to process any payouts Assert.Empty(await adminClient.ShowOnChainWalletTransactions(admin.StoreId, "BTC")); - await adminClient.RemovePayoutProcessor(admin.StoreId, tpGen.Name, tpGen.PaymentMethods.First()); + await adminClient.RemovePayoutProcessor(admin.StoreId, tpGen.Name, tpGen.PayoutMethods.First()); Assert.Empty(await adminClient.GetStoreOnChainAutomatedPayoutProcessors(admin.StoreId, "BTC")); Assert.Empty(await adminClient.GetPayoutProcessors(admin.StoreId)); diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 814d971d16..f350dcde4b 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -1463,7 +1463,7 @@ public async Task CanTopUpPullPayment() { Currency = "BTC", Amount = 1.0m, - PaymentMethods = [ "BTC-CHAIN" ] + PayoutMethods = [ "BTC-CHAIN" ] }); var controller = user.GetController(); var invoice = await controller.CreateInvoiceCoreRaw(new() @@ -1479,7 +1479,7 @@ await TestUtils.EventuallyAsync(async () => var payout = Assert.Single(payouts); Assert.Equal("TOPUP", payout.PayoutMethodId); Assert.Equal(invoice.Id, payout.Destination); - Assert.Equal(-0.5m, payout.Amount); + Assert.Equal(-0.5m, payout.OriginalAmount); }); } diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs index 6b41db362c..961847cde9 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs @@ -446,7 +446,7 @@ public async Task RefundInvoice( Name = request.Name ?? $"Refund {invoice.Id}", Description = request.Description, StoreId = storeId, - PayoutMethodIds = new[] { payoutMethodId }, + PayoutMethods = new[] { payoutMethodId }, }; if (request.RefundVariant != RefundVariant.Custom) diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldPayoutProcessorsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldPayoutProcessorsController.cs index 7d7a185de7..e37ae7fafe 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldPayoutProcessorsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldPayoutProcessorsController.cs @@ -36,7 +36,7 @@ public IActionResult GetPayoutProcessors() { Name = factory.Processor, FriendlyName = factory.FriendlyName, - PaymentMethods = factory.GetSupportedPayoutMethods().Select(id => id.ToString()) + PayoutMethods = factory.GetSupportedPayoutMethods().Select(id => id.ToString()) .ToArray() })); } diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldPullPaymentController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldPullPaymentController.cs index a2d25b5b05..c20032afd3 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldPullPaymentController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldPullPaymentController.cs @@ -132,7 +132,7 @@ public async Task CreatePullPayment(string storeId, CreatePullPay ModelState.AddModelError(nameof(request.BOLT11Expiration), $"The BOLT11 expiration should be positive"); } PayoutMethodId?[]? payoutMethods = null; - if (request.PaymentMethods is { } payoutMethodsStr) + if (request.PayoutMethods is { } payoutMethodsStr) { payoutMethods = payoutMethodsStr.Select(s => { @@ -144,13 +144,13 @@ public async Task CreatePullPayment(string storeId, CreatePullPay { if (!supported.Contains(payoutMethods[i])) { - request.AddModelError(paymentRequest => paymentRequest.PaymentMethods[i], "Invalid or unsupported payment method", this); + request.AddModelError(paymentRequest => paymentRequest.PayoutMethods[i], "Invalid or unsupported payment method", this); } } } else { - ModelState.AddModelError(nameof(request.PaymentMethods), "This field is required"); + ModelState.AddModelError(nameof(request.PayoutMethods), "This field is required"); } if (!ModelState.IsValid) return this.CreateValidationError(ModelState); @@ -364,16 +364,17 @@ private Client.Models.PayoutData ToModel(Data.PayoutData p) Id = p.Id, PullPaymentId = p.PullPaymentDataId, Date = p.Date, - Amount = p.OriginalAmount, - PaymentMethodAmount = p.Amount, + OriginalCurrency = p.OriginalCurrency, + OriginalAmount = p.OriginalAmount, + PayoutCurrency = p.Currency, + PayoutAmount = p.Amount, Revision = blob.Revision, State = p.State, + PayoutMethodId = p.PayoutMethodId, + PaymentProof = p.GetProofBlobJson(), + Destination = blob.Destination, Metadata = blob.Metadata?? new JObject(), }; - model.Destination = blob.Destination; - model.PayoutMethodId = p.PayoutMethodId; - model.CryptoCode = p.Currency; - model.PaymentProof = p.GetProofBlobJson(); return model; } diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedLightningPayoutProcessorsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedLightningPayoutProcessorsController.cs index 6d42ea9aca..7c47b845b7 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedLightningPayoutProcessorsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedLightningPayoutProcessorsController.cs @@ -46,7 +46,7 @@ await _payoutProcessorService.GetProcessors( { Stores = new[] { storeId }, Processors = new[] { LightningAutomatedPayoutSenderFactory.ProcessorName }, - PayoutMethodIds = paymentMethodId is null ? null : new[] { paymentMethodId } + PayoutMethods = paymentMethodId is null ? null : new[] { paymentMethodId } }); return Ok(configured.Select(ToModel).ToArray()); @@ -88,7 +88,7 @@ public async Task UpdateStoreLightningAutomatedPayoutProcessor( { Stores = new[] { storeId }, Processors = new[] { LightningAutomatedPayoutSenderFactory.ProcessorName }, - PayoutMethodIds = new[] { pmi } + PayoutMethods = new[] { pmi } })) .FirstOrDefault(); activeProcessor ??= new PayoutProcessorData(); diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedOnChainPayoutProcessorsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedOnChainPayoutProcessorsController.cs index f98af8e4e1..2298e8a5cf 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedOnChainPayoutProcessorsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStoreAutomatedOnChainPayoutProcessorsController.cs @@ -47,7 +47,7 @@ await _payoutProcessorService.GetProcessors( { Stores = new[] { storeId }, Processors = new[] { OnChainAutomatedPayoutSenderFactory.ProcessorName }, - PayoutMethodIds = paymentMethodId is null ? null : new[] { paymentMethodId } + PayoutMethods = paymentMethodId is null ? null : new[] { paymentMethodId } }); return Ok(configured.Select(ToModel).ToArray()); @@ -94,7 +94,7 @@ public async Task UpdateStoreOnchainAutomatedPayoutProcessor( { Stores = new[] { storeId }, Processors = new[] { OnChainAutomatedPayoutSenderFactory.ProcessorName }, - PayoutMethodIds = new[] { payoutMethodId } + PayoutMethods = new[] { payoutMethodId } })) .FirstOrDefault(); activeProcessor ??= new PayoutProcessorData(); diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStorePayoutProcessorsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStorePayoutProcessorsController.cs index 8b27ed061c..be9b0e6e62 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStorePayoutProcessorsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStorePayoutProcessorsController.cs @@ -39,7 +39,7 @@ public async Task GetStorePayoutProcessors( { Name = datas.Key, FriendlyName = _factories.FirstOrDefault(factory => factory.Processor == datas.Key)?.FriendlyName, - PaymentMethods = datas.Select(data => data.PayoutMethodId).ToArray() + PayoutMethods = datas.Select(data => data.PayoutMethodId).ToArray() }); return Ok(configured); @@ -55,7 +55,7 @@ public async Task RemoveStorePayoutProcessor( { Stores = new[] { storeId }, Processors = new[] { processor }, - PayoutMethodIds = new[] { PayoutMethodId.Parse(paymentMethod) } + PayoutMethods = new[] { PayoutMethodId.Parse(paymentMethod) } })).FirstOrDefault(); if (matched is null) { diff --git a/BTCPayServer/Controllers/UIInvoiceController.UI.cs b/BTCPayServer/Controllers/UIInvoiceController.UI.cs index 5416cce29b..d33d96a497 100644 --- a/BTCPayServer/Controllers/UIInvoiceController.UI.cs +++ b/BTCPayServer/Controllers/UIInvoiceController.UI.cs @@ -413,7 +413,7 @@ public async Task Refund(string invoiceId, RefundModel model, Can createPullPayment = new CreatePullPayment { Name = $"Refund {invoice.Id}", - PayoutMethodIds = new[] { pmi }, + PayoutMethods = new[] { pmi }, StoreId = invoice.StoreId, BOLT11Expiration = store.GetStoreBlob().RefundBOLT11Expiration }; diff --git a/BTCPayServer/Controllers/UIStorePullPaymentsController.PullPayments.cs b/BTCPayServer/Controllers/UIStorePullPaymentsController.PullPayments.cs index a0cb1f8e9a..66ede3aeb1 100644 --- a/BTCPayServer/Controllers/UIStorePullPaymentsController.PullPayments.cs +++ b/BTCPayServer/Controllers/UIStorePullPaymentsController.PullPayments.cs @@ -150,7 +150,7 @@ await _pullPaymentService.CreatePullPayment(new CreatePullPayment Amount = model.Amount, Currency = model.Currency, StoreId = storeId, - PayoutMethodIds = selectedPaymentMethodIds, + PayoutMethods = selectedPaymentMethodIds, BOLT11Expiration = TimeSpan.FromDays(model.BOLT11Expiration), AutoApproveClaims = model.AutoApproveClaims }); @@ -586,7 +586,7 @@ public async Task Payouts( private async Task HasPayoutProcessor(string storeId, PayoutMethodId payoutMethodId) { var processors = await _payoutProcessorService.GetProcessors( - new PayoutProcessorService.PayoutProcessorQuery { Stores = [storeId], PayoutMethodIds = [payoutMethodId] }); + new PayoutProcessorService.PayoutProcessorQuery { Stores = [storeId], PayoutMethods = [payoutMethodId] }); return _payoutProcessorFactories.Any(factory => factory.GetSupportedPayoutMethods().Contains(payoutMethodId)) && processors.Any(); } private async Task HasPayoutProcessor(string storeId, string payoutMethodId) diff --git a/BTCPayServer/HostedServices/PullPaymentHostedService.cs b/BTCPayServer/HostedServices/PullPaymentHostedService.cs index a8902116f8..14e79519e4 100644 --- a/BTCPayServer/HostedServices/PullPaymentHostedService.cs +++ b/BTCPayServer/HostedServices/PullPaymentHostedService.cs @@ -39,7 +39,7 @@ public class CreatePullPayment public string Description { get; set; } public decimal Amount { get; set; } public string Currency { get; set; } - public PayoutMethodId[] PayoutMethodIds { get; set; } + public PayoutMethodId[] PayoutMethods { get; set; } public bool AutoApproveClaims { get; set; } public TimeSpan? BOLT11Expiration { get; set; } } @@ -119,7 +119,7 @@ public Task CreatePullPayment(string storeId, CreatePullPaymentRequest r Amount = request.Amount, Currency = request.Currency, StoreId = storeId, - PayoutMethodIds = request.PaymentMethods.Select(p => PayoutMethodId.Parse(p)).ToArray(), + PayoutMethods = request.PayoutMethods.Select(p => PayoutMethodId.Parse(p)).ToArray(), AutoApproveClaims = request.AutoApproveClaims }); } @@ -143,7 +143,7 @@ public async Task CreatePullPayment(CreatePullPayment create) { Name = create.Name ?? string.Empty, Description = create.Description ?? string.Empty, - SupportedPayoutMethods = create.PayoutMethodIds, + SupportedPayoutMethods = create.PayoutMethods, AutoApproveClaims = create.AutoApproveClaims, View = new PullPaymentBlob.PullPaymentView { diff --git a/BTCPayServer/Models/WalletViewModels/PullPaymentsModel.cs b/BTCPayServer/Models/WalletViewModels/PullPaymentsModel.cs index 07db2c61fe..b7f234b5c2 100644 --- a/BTCPayServer/Models/WalletViewModels/PullPaymentsModel.cs +++ b/BTCPayServer/Models/WalletViewModels/PullPaymentsModel.cs @@ -37,8 +37,6 @@ public class ProgressModel public List PullPayments { get; set; } = new List(); public override int CurrentPageCount => PullPayments.Count; - public string PaymentMethodId { get; set; } - public IEnumerable PaymentMethods { get; set; } public PullPaymentState ActiveState { get; set; } = PullPaymentState.Active; } diff --git a/BTCPayServer/PayoutProcessors/Lightning/UILightningAutomatedPayoutProcessorsController.cs b/BTCPayServer/PayoutProcessors/Lightning/UILightningAutomatedPayoutProcessorsController.cs index e35af7a96c..d36a522224 100644 --- a/BTCPayServer/PayoutProcessors/Lightning/UILightningAutomatedPayoutProcessorsController.cs +++ b/BTCPayServer/PayoutProcessors/Lightning/UILightningAutomatedPayoutProcessorsController.cs @@ -54,7 +54,7 @@ public async Task Configure(string storeId, string cryptoCode) { Stores = new[] { storeId }, Processors = new[] { _lightningAutomatedPayoutSenderFactory.Processor }, - PayoutMethodIds = new[] + PayoutMethods = new[] { PayoutTypes.LN.GetPayoutMethodId(cryptoCode) } @@ -88,7 +88,7 @@ public async Task Configure(string storeId, string cryptoCode, Li { Stores = new[] { storeId }, Processors = new[] { _lightningAutomatedPayoutSenderFactory.Processor }, - PayoutMethodIds = new[] + PayoutMethods = new[] { PayoutTypes.LN.GetPayoutMethodId(cryptoCode) } diff --git a/BTCPayServer/PayoutProcessors/OnChain/UIOnChainAutomatedPayoutProcessorsController.cs b/BTCPayServer/PayoutProcessors/OnChain/UIOnChainAutomatedPayoutProcessorsController.cs index 988794e008..84a387dcb0 100644 --- a/BTCPayServer/PayoutProcessors/OnChain/UIOnChainAutomatedPayoutProcessorsController.cs +++ b/BTCPayServer/PayoutProcessors/OnChain/UIOnChainAutomatedPayoutProcessorsController.cs @@ -65,7 +65,7 @@ public async Task Configure(string storeId, string cryptoCode) { Stores = new[] { storeId }, Processors = new[] { _onChainAutomatedPayoutSenderFactory.Processor }, - PayoutMethodIds = new[] + PayoutMethods = new[] { PayoutTypes.CHAIN.GetPayoutMethodId(cryptoCode) } @@ -98,7 +98,7 @@ public async Task Configure(string storeId, string cryptoCode, On { Stores = new[] { storeId }, Processors = new[] { OnChainAutomatedPayoutSenderFactory.ProcessorName }, - PayoutMethodIds = new[] + PayoutMethods = new[] { PayoutTypes.CHAIN.GetPayoutMethodId(cryptoCode) } diff --git a/BTCPayServer/PayoutProcessors/PayoutProcessorService.cs b/BTCPayServer/PayoutProcessors/PayoutProcessorService.cs index d62024fd26..1733ed81a0 100644 --- a/BTCPayServer/PayoutProcessors/PayoutProcessorService.cs +++ b/BTCPayServer/PayoutProcessors/PayoutProcessorService.cs @@ -53,11 +53,11 @@ public PayoutProcessorQuery() public PayoutProcessorQuery(string storeId, PayoutMethodId payoutMethodId) { Stores = new[] { storeId }; - PayoutMethodIds = new[] { payoutMethodId }; + PayoutMethods = new[] { payoutMethodId }; } public string[] Stores { get; set; } public string[] Processors { get; set; } - public PayoutMethodId[] PayoutMethodIds { get; set; } + public PayoutMethodId[] PayoutMethods { get; set; } } public async Task> GetProcessors(PayoutProcessorQuery query) @@ -73,9 +73,9 @@ public async Task> GetProcessors(PayoutProcessorQuery { queryable = queryable.Where(data => query.Stores.Contains(data.StoreId)); } - if (query.PayoutMethodIds is not null) + if (query.PayoutMethods is not null) { - var paymentMethods = query.PayoutMethodIds.Select(d => d.ToString()).Distinct().ToArray(); + var paymentMethods = query.PayoutMethods.Select(d => d.ToString()).Distinct().ToArray(); queryable = queryable.Where(data => paymentMethods.Contains(data.PayoutMethodId)); } diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.payout-processors.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.payout-processors.json index d3ddfc60ac..0bcfb0452a 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.payout-processors.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.payout-processors.json @@ -531,7 +531,7 @@ "description": "Human name of the payout processor", "type": "string" }, - "paymentMethods": { + "payoutMethods": { "nullable": true, "description": "Supported, payment methods by this processor", "type": "array", diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json index b8bf79ff3f..b6ed313449 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json @@ -223,9 +223,9 @@ "nullable": true, "description": "When this pull payment expires. Never expires if null or unspecified." }, - "paymentMethods": { + "payoutMethods": { "type": "array", - "description": "The list of supported payment methods supported by this pull payment. Available options can be queried from the `StorePaymentMethods_GetStorePaymentMethods` endpoint", + "description": "The list of supported payout methods supported by this pull payment. Available options can be queried from the `StorePaymentMethods_GetStorePaymentMethods` endpoint", "items": { "type": "string", "example": "BTC" @@ -1091,11 +1091,29 @@ "example": "1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2", "description": "The destination of the payout (can be an address or a BIP21 url)" }, - "amount": { + "originalCurrency": { + "type": "string", + "example": "USD", + "description": "The currency before being converted into the payout's currency" + }, + "originalAmount": { "type": "string", "format": "decimal", + "nullable": false, "example": "10399.18", - "description": "The amount of the payout in the currency of the pull payment (eg. USD)." + "description": "The amount in originalCurrency before being converted into the payout's currency" + }, + "payoutCurrency": { + "type": "string", + "example": "BTC", + "description": "The currency of the payout after conversion." + }, + "payoutAmount": { + "type": "string", + "format": "decimal", + "nullable": true, + "example": "0.1", + "description": "The amount in payoutCurrency after conversion. (This property is set after the payout has been Approved)" }, "payoutMethodId": { "$ref": "#/components/schemas/PayoutMethodId" From 7013e618ded88ea056bb838f1f48c64ca0f1794b Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Thu, 26 Sep 2024 12:23:41 +0900 Subject: [PATCH 20/26] Remove dead fields from swagger --- .../swagger/v1/swagger.template.pull-payments.json | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json index b6ed313449..3eb546cbbc 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.pull-payments.json @@ -1112,24 +1112,12 @@ "type": "string", "format": "decimal", "nullable": true, - "example": "0.1", + "example": "1.12300000", "description": "The amount in payoutCurrency after conversion. (This property is set after the payout has been Approved)" }, "payoutMethodId": { "$ref": "#/components/schemas/PayoutMethodId" }, - "cryptoCode": { - "type": "string", - "example": "BTC", - "description": "Crypto code of the payment method of the payout (e.g., \"BTC\" or \"LTC\")" - }, - "paymentMethodAmount": { - "type": "string", - "format": "decimal", - "nullable": true, - "example": "1.12300000", - "description": "The amount of the payout in the currency of the payment method (eg. BTC). This is only available from the `AwaitingPayment` state." - }, "state": { "$ref": "#/components/schemas/PayoutState" }, From 443a350badaefc890846472d1feb7f713c9be94c Mon Sep 17 00:00:00 2001 From: d11n Date: Thu, 26 Sep 2024 08:52:16 +0200 Subject: [PATCH 21/26] App Service: Validate IDs when parsing items template (#6228) Validates missing and duplicate IDs on the edit actions and when creating/updating apps via the API. Fails gracefully by excluding existing items without ID or with duplicate ID for the rest of the cases. Fixes #6227. --- BTCPayServer.Tests/GreenfieldAPITests.cs | 48 +++++++++++++++++- BTCPayServer.Tests/POSTests.cs | 49 +++++++++++++++++++ BTCPayServer.Tests/SeleniumTests.cs | 9 ++++ .../GreenField/GreenfieldAppsController.cs | 12 ++--- .../Controllers/UICrowdfundController.cs | 6 +-- .../Controllers/UIPointOfSaleController.cs | 6 +-- BTCPayServer/Services/Apps/AppService.cs | 15 ++++-- .../Shared/Crowdfund/UpdateCrowdfund.cshtml | 2 +- .../PointOfSale/UpdatePointOfSale.cshtml | 2 +- .../Views/Shared/TemplateEditor.cshtml | 5 +- 10 files changed, 130 insertions(+), 24 deletions(-) diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index 8eb6fdf802..1a013eee5b 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -368,6 +368,27 @@ await AssertValidationError(new[] { "AppName", "Currency", "Template" }, } ) ); + var template = @"[ + { + ""description"": ""Lovely, fresh and tender, Meng Ding Gan Lu ('sweet dew') is grown in the lush Meng Ding Mountains of the southwestern province of Sichuan where it has been cultivated for over a thousand years."", + ""id"": ""green-tea"", + ""image"": ""~/img/pos-sample/green-tea.jpg"", + ""priceType"": ""Fixed"", + ""price"": ""1"", + ""title"": ""Green Tea"", + ""disabled"": false + } + ]"; + await AssertValidationError(new[] { "Template" }, + async () => await client.CreatePointOfSaleApp( + user.StoreId, + new PointOfSaleAppRequest + { + AppName = "good name", + Template = template.Replace(@"""id"": ""green-tea"",", "") + } + ) + ); // Test creating a POS app successfully var app = await client.CreatePointOfSaleApp( @@ -376,7 +397,8 @@ await AssertValidationError(new[] { "AppName", "Currency", "Template" }, { AppName = "test app from API", Currency = "JPY", - Title = "test app title" + Title = "test app title", + Template = template } ); Assert.Equal("test app from API", app.AppName); @@ -559,6 +581,27 @@ await AssertValidationError(new[] { "EndDate" }, } ) ); + var template = @"[ + { + ""description"": ""Lovely, fresh and tender, Meng Ding Gan Lu ('sweet dew') is grown in the lush Meng Ding Mountains of the southwestern province of Sichuan where it has been cultivated for over a thousand years."", + ""id"": ""green-tea"", + ""image"": ""~/img/pos-sample/green-tea.jpg"", + ""priceType"": ""Fixed"", + ""price"": ""1"", + ""title"": ""Green Tea"", + ""disabled"": false + } + ]"; + await AssertValidationError(new[] { "PerksTemplate" }, + async () => await client.CreateCrowdfundApp( + user.StoreId, + new CrowdfundAppRequest + { + AppName = "good name", + PerksTemplate = template.Replace(@"""id"": ""green-tea"",", "") + } + ) + ); // Test creating a crowdfund app var app = await client.CreateCrowdfundApp( @@ -566,7 +609,8 @@ await AssertValidationError(new[] { "EndDate" }, new CrowdfundAppRequest { AppName = "test app from API", - Title = "test app title" + Title = "test app title", + PerksTemplate = template } ); Assert.Equal("test app from API", app.AppName); diff --git a/BTCPayServer.Tests/POSTests.cs b/BTCPayServer.Tests/POSTests.cs index 5e4a4b8c53..b4a1b447c2 100644 --- a/BTCPayServer.Tests/POSTests.cs +++ b/BTCPayServer.Tests/POSTests.cs @@ -1,3 +1,4 @@ +using System; using System.Threading.Tasks; using BTCPayServer.Client; using BTCPayServer.Controllers; @@ -90,6 +91,54 @@ public void CanParseOldYmlCorrectly() Assert.Null( parsedDefault[4].AdditionalData); Assert.Null( parsedDefault[4].PaymentMethods); } + + [Fact] + [Trait("Fast", "Fast")] + public void CanParseAppTemplate() + { + var template = @"[ + { + ""description"": ""Lovely, fresh and tender, Meng Ding Gan Lu ('sweet dew') is grown in the lush Meng Ding Mountains of the southwestern province of Sichuan where it has been cultivated for over a thousand years."", + ""id"": ""green-tea"", + ""image"": ""~/img/pos-sample/green-tea.jpg"", + ""priceType"": ""Fixed"", + ""price"": ""1"", + ""title"": ""Green Tea"", + ""disabled"": false + }, + { + ""description"": ""Tian Jian Tian Jian means 'heavenly tippy tea' in Chinese, and it describes the finest grade of dark tea. Our Tian Jian dark tea is from Hunan province which is famous for making some of the best dark teas available."", + ""id"": ""black-tea"", + ""image"": ""~/img/pos-sample/black-tea.jpg"", + ""priceType"": ""Fixed"", + ""price"": ""1"", + ""title"": ""Black Tea"", + ""disabled"": false + } + ]"; + + var items = AppService.Parse(template); + Assert.Equal(2, items.Length); + Assert.Equal("green-tea", items[0].Id); + Assert.Equal("black-tea", items[1].Id); + + // Fails gracefully for missing ID + var missingId = template.Replace(@"""id"": ""green-tea"",", ""); + items = AppService.Parse(missingId); + Assert.Single(items); + Assert.Equal("black-tea", items[0].Id); + + // Throws for missing ID + Assert.Throws(() => AppService.Parse(missingId, true, true)); + + // Fails gracefully for duplicate IDs + var duplicateId = template.Replace(@"""id"": ""green-tea"",", @"""id"": ""black-tea"","); + items = AppService.Parse(duplicateId); + Assert.Empty(items); + + // Throws for duplicate IDs + Assert.Throws(() => AppService.Parse(duplicateId, true, true)); + } [Fact(Timeout = LongRunningTestTimeout)] [Trait("Integration", "Integration")] diff --git a/BTCPayServer.Tests/SeleniumTests.cs b/BTCPayServer.Tests/SeleniumTests.cs index 15fa473a8d..82728b1024 100644 --- a/BTCPayServer.Tests/SeleniumTests.cs +++ b/BTCPayServer.Tests/SeleniumTests.cs @@ -1253,6 +1253,15 @@ public async Task CanCreateAppPoS() s.ClickPagePrimary(); Assert.Contains("App updated", s.FindAlertMessage().Text); + + s.Driver.ScrollTo(By.Id("CodeTabButton")); + s.Driver.FindElement(By.Id("CodeTabButton")).Click(); + template = s.Driver.FindElement(By.Id("TemplateConfig")).GetAttribute("value"); + s.Driver.FindElement(By.Id("TemplateConfig")).Clear(); + s.Driver.FindElement(By.Id("TemplateConfig")).SendKeys(template.Replace(@"""id"": ""green-tea"",", "")); + + s.ClickPagePrimary(); + Assert.Contains("Invalid template: Missing ID for item \"Green Tea\".", s.Driver.FindElement(By.CssSelector(".validation-summary-errors")).Text); s.Driver.FindElement(By.Id("ViewApp")).Click(); var windows = s.Driver.WindowHandles; diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldAppsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldAppsController.cs index fe477403eb..e17a7fb165 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldAppsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldAppsController.cs @@ -402,11 +402,11 @@ private void ValidatePOSAppRequest(PointOfSaleAppRequest request) try { // Just checking if we can serialize - AppService.SerializeTemplate(AppService.Parse(request.Template)); + AppService.SerializeTemplate(AppService.Parse(request.Template, true, true)); } - catch + catch (Exception ex) { - ModelState.AddModelError(nameof(request.Template), "Invalid template"); + ModelState.AddModelError(nameof(request.Template), ex.Message); } } } @@ -486,11 +486,11 @@ private void ValidateCrowdfundAppRequest(CrowdfundAppRequest request) try { // Just checking if we can serialize - AppService.SerializeTemplate(AppService.Parse(request.PerksTemplate)); + AppService.SerializeTemplate(AppService.Parse(request.PerksTemplate, true, true)); } - catch + catch (Exception ex) { - ModelState.AddModelError(nameof(request.PerksTemplate), "Invalid template"); + ModelState.AddModelError(nameof(request.PerksTemplate), $"Invalid template: {ex.Message}"); } } diff --git a/BTCPayServer/Plugins/Crowdfund/Controllers/UICrowdfundController.cs b/BTCPayServer/Plugins/Crowdfund/Controllers/UICrowdfundController.cs index a1edaa088b..321eab3923 100644 --- a/BTCPayServer/Plugins/Crowdfund/Controllers/UICrowdfundController.cs +++ b/BTCPayServer/Plugins/Crowdfund/Controllers/UICrowdfundController.cs @@ -447,11 +447,11 @@ public async Task UpdateCrowdfund(string appId, UpdateCrowdfundVi try { - vm.PerksTemplate = AppService.SerializeTemplate(AppService.Parse(vm.PerksTemplate)); + vm.PerksTemplate = AppService.SerializeTemplate(AppService.Parse(vm.PerksTemplate, true, true)); } - catch + catch (Exception ex) { - ModelState.AddModelError(nameof(vm.PerksTemplate), "Invalid template"); + ModelState.AddModelError(nameof(vm.PerksTemplate), $"Invalid template: {ex.Message}"); } if (vm.TargetAmount is decimal v && v == 0.0m) { diff --git a/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs b/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs index 27202be977..7f5cae7f39 100644 --- a/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs +++ b/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs @@ -647,11 +647,11 @@ public async Task UpdatePointOfSale(string appId, UpdatePointOfSa ModelState.AddModelError(nameof(vm.Currency), "Invalid currency"); try { - vm.Template = AppService.SerializeTemplate(AppService.Parse(vm.Template)); + vm.Template = AppService.SerializeTemplate(AppService.Parse(vm.Template, true, true)); } - catch + catch (Exception ex) { - ModelState.AddModelError(nameof(vm.Template), "Invalid template"); + ModelState.AddModelError(nameof(vm.Template), $"Invalid template: {ex.Message}"); } if (!ModelState.IsValid) { diff --git a/BTCPayServer/Services/Apps/AppService.cs b/BTCPayServer/Services/Apps/AppService.cs index c4fd79f8f6..b10229a911 100644 --- a/BTCPayServer/Services/Apps/AppService.cs +++ b/BTCPayServer/Services/Apps/AppService.cs @@ -348,12 +348,17 @@ public static string SerializeTemplate(ViewPointOfSaleViewModel.Item[] items) { return JsonConvert.SerializeObject(items, Formatting.Indented, _defaultSerializer); } - public static ViewPointOfSaleViewModel.Item[] Parse(string template, bool includeDisabled = true) + public static ViewPointOfSaleViewModel.Item[] Parse(string template, bool includeDisabled = true, bool throws = false) { - if (string.IsNullOrWhiteSpace(template)) - return Array.Empty(); - - return JsonConvert.DeserializeObject(template, _defaultSerializer)!.Where(item => includeDisabled || !item.Disabled).ToArray(); + if (string.IsNullOrWhiteSpace(template)) return []; + var allItems = JsonConvert.DeserializeObject(template, _defaultSerializer)!; + // ensure all items have an id, which is also unique + var itemsWithoutId = allItems.Where(i => string.IsNullOrEmpty(i.Id)).ToList(); + if (itemsWithoutId.Any() && throws) throw new ArgumentException($"Missing ID for item \"{itemsWithoutId.First().Title}\"."); + // find items with duplicate IDs + var duplicateIds = allItems.GroupBy(i => i.Id).Where(g => g.Count() > 1).Select(g => g.Key).ToList(); + if (duplicateIds.Any() && throws) throw new ArgumentException($"Duplicate ID \"{duplicateIds.First()}\"."); + return allItems.Where(item => (includeDisabled || !item.Disabled) && !itemsWithoutId.Contains(item) && !duplicateIds.Contains(item.Id)).ToArray(); } #nullable restore #nullable enable diff --git a/BTCPayServer/Views/Shared/Crowdfund/UpdateCrowdfund.cshtml b/BTCPayServer/Views/Shared/Crowdfund/UpdateCrowdfund.cshtml index 203d133006..3bf9c2b01f 100644 --- a/BTCPayServer/Views/Shared/Crowdfund/UpdateCrowdfund.cshtml +++ b/BTCPayServer/Views/Shared/Crowdfund/UpdateCrowdfund.cshtml @@ -48,7 +48,7 @@ @if (!ViewContext.ModelState.IsValid) { -
+
}
diff --git a/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml b/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml index 9dbda45247..16c5958b85 100644 --- a/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml +++ b/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml @@ -61,7 +61,7 @@ @if (!ViewContext.ModelState.IsValid) { -
+
}
diff --git a/BTCPayServer/Views/Shared/TemplateEditor.cshtml b/BTCPayServer/Views/Shared/TemplateEditor.cshtml index e542805d4e..567d286ddb 100644 --- a/BTCPayServer/Views/Shared/TemplateEditor.cshtml +++ b/BTCPayServer/Views/Shared/TemplateEditor.cshtml @@ -118,13 +118,12 @@
-

@Model.title

+

@Model.title

@if (ViewContext.ViewData.ModelState.TryGetValue(Model.templateId, out var errors)) { foreach (var error in errors.Errors) { -
- @error.ErrorMessage +

@error.ErrorMessage

} }
From b5590a38feece01f14681344d55e3522b88bdac5 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Thu, 26 Sep 2024 19:09:12 +0900 Subject: [PATCH 22/26] Add better error message if v1 routes are used. --- BTCPayServer.Tests/GreenfieldAPITests.cs | 3 ++ .../GreenfieldObsoleteController.cs | 43 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 BTCPayServer/Controllers/GreenField/GreenfieldObsoleteController.cs diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index 1a013eee5b..987f8a6600 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -97,6 +97,9 @@ public async Task MissingPermissionTest() Assert.NotNull(e.APIError.Message); GreenfieldPermissionAPIError permissionError = Assert.IsType(e.APIError); Assert.Equal(Policies.CanModifyStoreSettings, permissionError.MissingPermission); + + var client = await user.CreateClient(Policies.CanViewStoreSettings); + await AssertAPIError("unsupported-in-v2", () => client.SendHttpRequest($"api/v1/stores/{user.StoreId}/payment-methods/LightningNetwork")); } [Fact(Timeout = TestTimeout)] diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldObsoleteController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldObsoleteController.cs new file mode 100644 index 0000000000..c70e74bced --- /dev/null +++ b/BTCPayServer/Controllers/GreenField/GreenfieldObsoleteController.cs @@ -0,0 +1,43 @@ +using BTCPayServer.Abstractions.Constants; +using BTCPayServer.Abstractions.Extensions; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Cors; +using Microsoft.AspNetCore.Mvc; + +namespace BTCPayServer.Controllers.GreenField +{ + [ApiController] + [EnableCors(CorsPolicies.All)] + [Authorize(AuthenticationSchemes = AuthenticationSchemes.Greenfield)] + public class GreenfieldObsoleteController : ControllerBase + { + [HttpGet("~/api/v1/stores/{storeId}/payment-methods/LNURL")] + public IActionResult Obsolete1(string storeId) + { + return Obsolete(); + } + [HttpGet("~/api/v1/stores/{storeId}/payment-methods/LNURLPay/{cryptoCode}")] + [HttpDelete("~/api/v1/stores/{storeId}/payment-methods/LNURLPay/{cryptoCode}")] + [HttpPut("~/api/v1/stores/{storeId}/payment-methods/LNURLPay/{cryptoCode}")] + public IActionResult Obsolete2(string storeId, string cryptoCode) + { + return Obsolete(); + } + [HttpGet("~/api/v1/stores/{storeId}/payment-methods/LightningNetwork")] + public IActionResult Obsolete3(string storeId) + { + return Obsolete(); + } + [HttpGet("~/api/v1/stores/{storeId}/payment-methods/LightningNetwork/{cryptoCode}")] + [HttpDelete("~/api/v1/stores/{storeId}/payment-methods/LightningNetwork/{cryptoCode}")] + [HttpPut("~/api/v1/stores/{storeId}/payment-methods/LightningNetwork/{cryptoCode}")] + public IActionResult Obsolete4(string storeId, string cryptoCode) + { + return Obsolete(); + } + private IActionResult Obsolete() + { + return this.CreateAPIError(410, "unsupported-in-v2", "This route isn't supported by BTCPay Server 2.0 and newer. Please update your integration."); + } + } +} From 272cc3d3c9caf97e538dfcc855ae2cee7c1c45fa Mon Sep 17 00:00:00 2001 From: d11n Date: Thu, 26 Sep 2024 12:10:14 +0200 Subject: [PATCH 23/26] POS: Option for user sign in via the QR code (#6231) * Login Code: Turn into Blazor component and extend with data for the app * POS: Add login code for POS frontend * Improve components, fix test --- BTCPayServer.Tests/SeleniumTests.cs | 11 +- BTCPayServer/Blazor/Icon.razor | 9 +- BTCPayServer/Blazor/PosLoginCode.razor | 42 ++++++++ BTCPayServer/Blazor/QrCode.razor | 29 +++++ BTCPayServer/Blazor/UserLoginCode.razor | 100 ++++++++++++++++++ .../Controllers/UIAccountController.cs | 17 ++- .../UIManageController.LoginCodes.cs | 21 ++-- .../Controllers/UIManageController.cs | 3 - .../Extensions/UrlHelperExtensions.cs | 16 ++- BTCPayServer/Fido2/UserLoginCodeService.cs | 5 +- .../Controllers/UIPointOfSaleController.cs | 9 ++ .../Models/UpdatePointOfSaleViewModel.cs | 1 + .../PointOfSale/UpdatePointOfSale.cshtml | 27 +++-- BTCPayServer/Views/UIManage/LoginCodes.cshtml | 36 +------ 14 files changed, 252 insertions(+), 74 deletions(-) create mode 100644 BTCPayServer/Blazor/PosLoginCode.razor create mode 100644 BTCPayServer/Blazor/QrCode.razor create mode 100644 BTCPayServer/Blazor/UserLoginCode.razor diff --git a/BTCPayServer.Tests/SeleniumTests.cs b/BTCPayServer.Tests/SeleniumTests.cs index 82728b1024..53c008d52b 100644 --- a/BTCPayServer.Tests/SeleniumTests.cs +++ b/BTCPayServer.Tests/SeleniumTests.cs @@ -3426,11 +3426,14 @@ public async Task CanSigninWithLoginCode() var user = s.RegisterNewUser(); s.GoToHome(); s.GoToProfile(ManageNavPages.LoginCodes); - var code = s.Driver.FindElement(By.Id("logincode")).GetAttribute("value"); - s.ClickPagePrimary(); - Assert.NotEqual(code, s.Driver.FindElement(By.Id("logincode")).GetAttribute("value")); - code = s.Driver.FindElement(By.Id("logincode")).GetAttribute("value"); + string code = null; + TestUtils.Eventually(() => { code = s.Driver.FindElement(By.CssSelector("#LoginCode .qr-code")).GetAttribute("alt"); }); + string prevCode = code; + await s.Driver.Navigate().RefreshAsync(); + TestUtils.Eventually(() => { code = s.Driver.FindElement(By.CssSelector("#LoginCode .qr-code")).GetAttribute("alt"); }); + Assert.NotEqual(prevCode, code); + TestUtils.Eventually(() => { code = s.Driver.FindElement(By.CssSelector("#LoginCode .qr-code")).GetAttribute("alt"); }); s.Logout(); s.GoToLogin(); s.Driver.SetAttribute("LoginCode", "value", "bad code"); diff --git a/BTCPayServer/Blazor/Icon.razor b/BTCPayServer/Blazor/Icon.razor index 3fca30bb41..584b36e386 100644 --- a/BTCPayServer/Blazor/Icon.razor +++ b/BTCPayServer/Blazor/Icon.razor @@ -7,12 +7,13 @@ @code { - public string GetPathTo(string symbol) + [Parameter, EditorRequired] + public string Symbol { get; set; } + + private string GetPathTo(string symbol) { var versioned = FileVersionProvider.AddFileVersionToPath(default, "img/icon-sprite.svg"); var rootPath = (BTCPayServerOptions.RootPath ?? "/").WithTrailingSlash(); - return $"{rootPath}{versioned}#{Symbol}"; + return $"{rootPath}{versioned}#{symbol}"; } - [Parameter] - public string Symbol { get; set; } } diff --git a/BTCPayServer/Blazor/PosLoginCode.razor b/BTCPayServer/Blazor/PosLoginCode.razor new file mode 100644 index 0000000000..42319562ba --- /dev/null +++ b/BTCPayServer/Blazor/PosLoginCode.razor @@ -0,0 +1,42 @@ +@using Microsoft.AspNetCore.Http + +@inject IHttpContextAccessor HttpContextAccessor; + +@if (Users?.Any() is true) +{ +
+ + +
+} + +@if (string.IsNullOrEmpty(_userId)) +{ + +} +else +{ + +} + +@code { + [Parameter, EditorRequired] + public string PosPath { get; set; } + + [Parameter] + public Dictionary Users { get; set; } + + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary Attrs { get; set; } + + private string _userId; + private string PosUrl => Request.GetAbsoluteRoot() + PosPath; + private HttpRequest Request => HttpContextAccessor.HttpContext?.Request; + private string CssClass => $"form-group {(Attrs?.ContainsKey("class") is true ? Attrs["class"] : "")}".Trim(); +} diff --git a/BTCPayServer/Blazor/QrCode.razor b/BTCPayServer/Blazor/QrCode.razor new file mode 100644 index 0000000000..ac94fe0dc0 --- /dev/null +++ b/BTCPayServer/Blazor/QrCode.razor @@ -0,0 +1,29 @@ +@using QRCoder + +@if (!string.IsNullOrEmpty(Data)) +{ + @Data +} + +@code { + [Parameter, EditorRequired] + public string Data { get; set; } + + [Parameter] + public int Size { get; set; } = 256; + + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary Attrs { get; set; } + + private static readonly QRCodeGenerator QrGenerator = new(); + + private string GetBase64(string data) + { + var qrCodeData = QrGenerator.CreateQrCode(data, QRCodeGenerator.ECCLevel.Q); + var qrCode = new PngByteQRCode(qrCodeData); + var bytes = qrCode.GetGraphic(5, [0, 0, 0, 255], [0xf5, 0xf5, 0xf7, 255]); + return Convert.ToBase64String(bytes); + } + + private string CssClass => $"qr-code {(Attrs?.ContainsKey("class") is true ? Attrs["class"] : "")}".Trim(); +} diff --git a/BTCPayServer/Blazor/UserLoginCode.razor b/BTCPayServer/Blazor/UserLoginCode.razor new file mode 100644 index 0000000000..c3fa74ecc3 --- /dev/null +++ b/BTCPayServer/Blazor/UserLoginCode.razor @@ -0,0 +1,100 @@ +@using System.Timers +@using BTCPayServer.Data +@using BTCPayServer.Fido2 +@using Microsoft.AspNetCore.Http +@using Microsoft.AspNetCore.Identity +@using Microsoft.AspNetCore.Mvc +@using Microsoft.AspNetCore.Routing +@inject AuthenticationStateProvider AuthenticationStateProvider +@inject UserManager UserManager; +@inject UserLoginCodeService UserLoginCodeService; +@inject LinkGenerator LinkGenerator; +@inject IHttpContextAccessor HttpContextAccessor; +@implements IDisposable + +@if (!string.IsNullOrEmpty(_data)) +{ +
+
+ +
+

Valid for @_seconds seconds

+
+
+
+
+} + +@code { + [Parameter] + public string UserId { get; set; } + + [Parameter] + public string RedirectUrl { get; set; } + + [Parameter] + public int Size { get; set; } = 256; + + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary Attrs { get; set; } + + private static readonly double Seconds = UserLoginCodeService.ExpirationTime.TotalSeconds; + private double _seconds = Seconds; + private string _data; + private ApplicationUser _user; + private Timer _timer; + + protected override async Task OnParametersSetAsync() + { + UserId ??= await GetUserId(); + if (!string.IsNullOrEmpty(UserId)) _user = await UserManager.FindByIdAsync(UserId); + if (_user == null) return; + + GenerateCodeAndStartTimer(); + } + + public void Dispose() + { + _timer?.Dispose(); + } + + private void GenerateCodeAndStartTimer() + { + var loginCode = UserLoginCodeService.GetOrGenerate(_user.Id); + _data = GetData(loginCode); + _seconds = Seconds; + _timer?.Dispose(); + _timer = new Timer(1000); + _timer.Elapsed += CountDownTimer; + _timer.Enabled = true; + } + + private void CountDownTimer(object source, ElapsedEventArgs e) + { + if (_seconds > 0) + _seconds -= 1; + else + GenerateCodeAndStartTimer(); + InvokeAsync(StateHasChanged); + } + + private async Task GetUserId() + { + var state = await AuthenticationStateProvider.GetAuthenticationStateAsync(); + return state.User.Identity?.IsAuthenticated is true + ? UserManager.GetUserId(state.User) + : null; + } + + private string GetData(string loginCode) + { + var req = HttpContextAccessor.HttpContext?.Request; + if (req == null) return loginCode; + return !string.IsNullOrEmpty(RedirectUrl) + ? LinkGenerator.LoginCodeLink(loginCode, RedirectUrl, req.Scheme, req.Host, req.PathBase) + : $"{loginCode};{LinkGenerator.IndexLink(req.Scheme, req.Host, req.PathBase)};{_user.Email}"; + } + + private double Percent => Math.Round(_seconds / Seconds * 100); + private string CssClass => $"user-login-code d-inline-flex flex-column {(Attrs?.ContainsKey("class") is true ? Attrs["class"] : "")}".Trim(); +} diff --git a/BTCPayServer/Controllers/UIAccountController.cs b/BTCPayServer/Controllers/UIAccountController.cs index 7971ded66b..f068594a47 100644 --- a/BTCPayServer/Controllers/UIAccountController.cs +++ b/BTCPayServer/Controllers/UIAccountController.cs @@ -123,15 +123,30 @@ public async Task Login(string returnUrl = null, string email = n return View(nameof(Login), new LoginViewModel { Email = email }); } + // GET is for signin via the POS backend + [HttpGet("/login/code")] + [AllowAnonymous] + [RateLimitsFilter(ZoneLimits.Login, Scope = RateLimitsScope.RemoteAddress)] + public async Task LoginUsingCode(string loginCode, string returnUrl = null) + { + return await LoginCodeResult(loginCode, returnUrl); + } + [HttpPost("/login/code")] [AllowAnonymous] [ValidateAntiForgeryToken] [RateLimitsFilter(ZoneLimits.Login, Scope = RateLimitsScope.RemoteAddress)] public async Task LoginWithCode(string loginCode, string returnUrl = null) + { + return await LoginCodeResult(loginCode, returnUrl); + } + + private async Task LoginCodeResult(string loginCode, string returnUrl) { if (!string.IsNullOrEmpty(loginCode)) { - var userId = _userLoginCodeService.Verify(loginCode); + var code = loginCode.Split(';').First(); + var userId = _userLoginCodeService.Verify(code); if (userId is null) { TempData[WellKnownTempData.ErrorMessage] = "Login code was invalid"; diff --git a/BTCPayServer/Controllers/UIManageController.LoginCodes.cs b/BTCPayServer/Controllers/UIManageController.LoginCodes.cs index 655085a813..39a9b3b7fb 100644 --- a/BTCPayServer/Controllers/UIManageController.LoginCodes.cs +++ b/BTCPayServer/Controllers/UIManageController.LoginCodes.cs @@ -1,21 +1,12 @@ -using System; -using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; -namespace BTCPayServer.Controllers +namespace BTCPayServer.Controllers; + +public partial class UIManageController { - public partial class UIManageController + [HttpGet] + public ActionResult LoginCodes() { - [HttpGet] - public async Task LoginCodes() - { - var user = await _userManager.GetUserAsync(User); - if (user == null) - { - throw new ApplicationException($"Unable to load user with ID '{_userManager.GetUserId(User)}'."); - } - - return View(nameof(LoginCodes), _userLoginCodeService.GetOrGenerate(user.Id)); - } + return View(); } } diff --git a/BTCPayServer/Controllers/UIManageController.cs b/BTCPayServer/Controllers/UIManageController.cs index b111b5551b..fe1bce45b4 100644 --- a/BTCPayServer/Controllers/UIManageController.cs +++ b/BTCPayServer/Controllers/UIManageController.cs @@ -40,7 +40,6 @@ public partial class UIManageController : Controller private readonly IAuthorizationService _authorizationService; private readonly Fido2Service _fido2Service; private readonly LinkGenerator _linkGenerator; - private readonly UserLoginCodeService _userLoginCodeService; private readonly IHtmlHelper Html; private readonly UserService _userService; private readonly UriResolver _uriResolver; @@ -62,7 +61,6 @@ public UIManageController( UserService userService, UriResolver uriResolver, IFileService fileService, - UserLoginCodeService userLoginCodeService, IHtmlHelper htmlHelper ) { @@ -76,7 +74,6 @@ IHtmlHelper htmlHelper _authorizationService = authorizationService; _fido2Service = fido2Service; _linkGenerator = linkGenerator; - _userLoginCodeService = userLoginCodeService; Html = htmlHelper; _userService = userService; _uriResolver = uriResolver; diff --git a/BTCPayServer/Extensions/UrlHelperExtensions.cs b/BTCPayServer/Extensions/UrlHelperExtensions.cs index 19d0eb713e..16e2a0aa51 100644 --- a/BTCPayServer/Extensions/UrlHelperExtensions.cs +++ b/BTCPayServer/Extensions/UrlHelperExtensions.cs @@ -3,7 +3,6 @@ using BTCPayServer; using BTCPayServer.Client.Models; using BTCPayServer.Controllers; -using BTCPayServer.Services.Apps; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; @@ -52,6 +51,12 @@ public static string LoginLink(this LinkGenerator urlHelper, string scheme, Host { return urlHelper.GetUriByAction(nameof(UIAccountController.Login), "UIAccount", null , scheme, host, pathbase); } + + public static string LoginCodeLink(this LinkGenerator urlHelper, string loginCode, string returnUrl, string scheme, HostString host, string pathbase) + { + return urlHelper.GetUriByAction(nameof(UIAccountController.LoginUsingCode), "UIAccount", new { loginCode, returnUrl }, scheme, host, pathbase); + } + public static string ResetPasswordLink(this LinkGenerator urlHelper, string userId, string code, string scheme, HostString host, string pathbase) { return urlHelper.GetUriByAction( @@ -109,5 +114,14 @@ public static string PayoutLink(this LinkGenerator urlHelper, string walletIdOrS values: new { storeId = wallet?.StoreId ?? walletIdOrStoreId, pullPaymentId, payoutState }, scheme, host, pathbase); } + + public static string IndexLink(this LinkGenerator urlHelper, string scheme, HostString host, string pathbase) + { + return urlHelper.GetUriByAction( + action: nameof(UIHomeController.Index), + controller: "UIHome", + values: null, + scheme, host, pathbase); + } } } diff --git a/BTCPayServer/Fido2/UserLoginCodeService.cs b/BTCPayServer/Fido2/UserLoginCodeService.cs index e84522884e..09c78b26b6 100644 --- a/BTCPayServer/Fido2/UserLoginCodeService.cs +++ b/BTCPayServer/Fido2/UserLoginCodeService.cs @@ -8,6 +8,7 @@ namespace BTCPayServer.Fido2 public class UserLoginCodeService { private readonly IMemoryCache _memoryCache; + public static readonly TimeSpan ExpirationTime = TimeSpan.FromSeconds(60); public UserLoginCodeService(IMemoryCache memoryCache) { @@ -29,10 +30,10 @@ public string GetOrGenerate(string userId) } return _memoryCache.GetOrCreate(GetCacheKey(userId), entry => { - entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1); + entry.AbsoluteExpirationRelativeToNow = ExpirationTime; var code = Encoders.Hex.EncodeData(RandomUtils.GetBytes(20)); using var newEntry = _memoryCache.CreateEntry(code); - newEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1); + newEntry.AbsoluteExpirationRelativeToNow = ExpirationTime; newEntry.Value = userId; return code; diff --git a/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs b/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs index 7f5cae7f39..54ef0d6c8d 100644 --- a/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs +++ b/BTCPayServer/Plugins/PointOfSale/Controllers/UIPointOfSaleController.cs @@ -627,6 +627,8 @@ public async Task UpdatePointOfSale(string appId) } vm.ExampleCallback = "{\n \"id\":\"SkdsDghkdP3D3qkj7bLq3\",\n \"url\":\"https://btcpay.example.com/invoice?id=SkdsDghkdP3D3qkj7bLq3\",\n \"status\":\"paid\",\n \"price\":10,\n \"currency\":\"EUR\",\n \"invoiceTime\":1520373130312,\n \"expirationTime\":1520374030312,\n \"currentTime\":1520373179327,\n \"exceptionStatus\":false,\n \"buyerFields\":{\n \"buyerEmail\":\"customer@example.com\",\n \"buyerNotify\":false\n },\n \"paymentSubtotals\": {\n \"BTC\":114700\n },\n \"paymentTotals\": {\n \"BTC\":118400\n },\n \"transactionCurrency\": \"BTC\",\n \"amountPaid\": \"1025900\",\n \"exchangeRates\": {\n \"BTC\": {\n \"EUR\": 8721.690715789999,\n \"USD\": 10817.99\n }\n }\n}"; + + await FillUsers(vm); return View("PointOfSale/UpdatePointOfSale", vm); } @@ -655,6 +657,7 @@ public async Task UpdatePointOfSale(string appId, UpdatePointOfSa } if (!ModelState.IsValid) { + await FillUsers(vm); return View("PointOfSale/UpdatePointOfSale", vm); } @@ -715,5 +718,11 @@ private async Task GetStoreDefaultCurrentIfEmpty(string storeId, string private StoreData GetCurrentStore() => HttpContext.GetStoreData(); private AppData GetCurrentApp() => HttpContext.GetAppData(); + + private async Task FillUsers(UpdatePointOfSaleViewModel vm) + { + var users = await _storeRepository.GetStoreUsers(GetCurrentStore().Id); + vm.StoreUsers = users.Select(u => (u.Id, u.Email, u.StoreRole.Role)).ToDictionary(u => u.Id, u => $"{u.Email} ({u.Role})"); + } } } diff --git a/BTCPayServer/Plugins/PointOfSale/Models/UpdatePointOfSaleViewModel.cs b/BTCPayServer/Plugins/PointOfSale/Models/UpdatePointOfSaleViewModel.cs index 2930a55df2..7d79ff246e 100644 --- a/BTCPayServer/Plugins/PointOfSale/Models/UpdatePointOfSaleViewModel.cs +++ b/BTCPayServer/Plugins/PointOfSale/Models/UpdatePointOfSaleViewModel.cs @@ -68,6 +68,7 @@ public class UpdatePointOfSaleViewModel public string CustomTipPercentages { get; set; } public string Id { get; set; } + public Dictionary StoreUsers { get; set; } [Display(Name = "Redirect invoice to redirect url automatically after paid")] public string RedirectAutomatically { get; set; } = string.Empty; diff --git a/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml b/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml index 16c5958b85..39447208f7 100644 --- a/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml +++ b/BTCPayServer/Views/Shared/PointOfSale/UpdatePointOfSale.cshtml @@ -12,6 +12,7 @@ ViewData.SetActivePage(AppsNavPages.Update, "Update Point of Sale", Model.Id); Csp.UnsafeEval(); var checkoutFormOptions = await FormDataService.GetSelect(Model.StoreId, Model.FormId); + var posPath = Url.Action("ViewPointOfSale", "UIPointOfSale", new { appId = Model.Id }); } @section PageHeadContent { @@ -23,14 +24,7 @@ @section PageFootContent { - - - }
@@ -46,7 +40,7 @@ {
View -
@@ -316,6 +310,19 @@ - - + diff --git a/BTCPayServer/Views/UIManage/LoginCodes.cshtml b/BTCPayServer/Views/UIManage/LoginCodes.cshtml index 478dc12dee..1429499d81 100644 --- a/BTCPayServer/Views/UIManage/LoginCodes.cshtml +++ b/BTCPayServer/Views/UIManage/LoginCodes.cshtml @@ -1,43 +1,11 @@ -@model string +@inject UserManager UserManager; @{ ViewData.SetActivePage(ManageNavPages.LoginCodes, "Login Codes"); }

@ViewData["Title"]

- Regenerate code

Easily log into BTCPay Server on another device using a simple login code from an already authenticated device.

-
-
- -
- -

Valid for 60 seconds

-
-
-
-
- -@section PageFootContent -{ - - - -} + From 9ba4b030ed88b28da8b228e9c85bc2a3e582c218 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Fri, 27 Sep 2024 15:27:04 +0900 Subject: [PATCH 24/26] Fix: Do not expose xpub without modify store permission (#6212) --- .../BTCPayServerClient.Invoices.cs | 8 +- BTCPayServer.Tests/GreenfieldAPITests.cs | 8 + .../GreenField/GreenfieldInvoiceController.cs | 134 ++++---------- ...GreenfieldStorePaymentMethodsController.cs | 4 +- .../GreenField/LocalBTCPayServerClient.cs | 6 +- .../Extensions/AuthorizationExtensions.cs | 6 + .../Bitcoin/BitcoinLikePaymentHandler.cs | 5 +- .../Bitcoin/BitcoinPaymentPromptDetails.cs | 7 + .../Payments/IPaymentMethodHandler.cs | 6 + .../swagger/v1/swagger.template.invoices.json | 170 +++++++++++------- 10 files changed, 185 insertions(+), 169 deletions(-) diff --git a/BTCPayServer.Client/BTCPayServerClient.Invoices.cs b/BTCPayServer.Client/BTCPayServerClient.Invoices.cs index 2e201beea1..ee58c8bb47 100644 --- a/BTCPayServer.Client/BTCPayServerClient.Invoices.cs +++ b/BTCPayServer.Client/BTCPayServerClient.Invoices.cs @@ -46,9 +46,15 @@ public virtual async Task GetInvoice(string storeId, string invoice return await SendHttpRequest($"api/v1/stores/{storeId}/invoices/{invoiceId}", null, HttpMethod.Get, token); } public virtual async Task GetInvoicePaymentMethods(string storeId, string invoiceId, + bool onlyAccountedPayments = true, bool includeSensitive = false, CancellationToken token = default) { - return await SendHttpRequest($"api/v1/stores/{storeId}/invoices/{invoiceId}/payment-methods", null, HttpMethod.Get, token); + var queryPayload = new Dictionary + { + { nameof(onlyAccountedPayments), onlyAccountedPayments }, + { nameof(includeSensitive), includeSensitive } + }; + return await SendHttpRequest($"api/v1/stores/{storeId}/invoices/{invoiceId}/payment-methods", queryPayload, HttpMethod.Get, token); } public virtual async Task ArchiveInvoice(string storeId, string invoiceId, diff --git a/BTCPayServer.Tests/GreenfieldAPITests.cs b/BTCPayServer.Tests/GreenfieldAPITests.cs index 987f8a6600..3d306d4cab 100644 --- a/BTCPayServer.Tests/GreenfieldAPITests.cs +++ b/BTCPayServer.Tests/GreenfieldAPITests.cs @@ -2421,6 +2421,14 @@ await TestUtils.EventuallyAsync(async () => invoice = await client.CreateInvoice(user.StoreId, new CreateInvoiceRequest { Amount = 5000.0m, Currency = "USD" }); methods = await client.GetInvoicePaymentMethods(user.StoreId, invoice.Id); method = methods.First(); + Assert.Equal(JTokenType.Null, method.AdditionalData["accountDerivation"].Type); + Assert.NotNull(method.AdditionalData["keyPath"]); + + methods = await client.GetInvoicePaymentMethods(user.StoreId, invoice.Id, includeSensitive: true); + method = methods.First(); + Assert.Equal(JTokenType.String, method.AdditionalData["accountDerivation"].Type); + var clientViewOnly = await user.CreateClient(Policies.CanViewInvoices); + await AssertApiError(403, "missing-permission", () => clientViewOnly.GetInvoicePaymentMethods(user.StoreId, invoice.Id, includeSensitive: true)); await tester.WaitForEvent(async () => { diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs index 961847cde9..c16f186c3d 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldInvoiceController.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -14,6 +15,7 @@ using BTCPayServer.Payments.Bitcoin; using BTCPayServer.Payouts; using BTCPayServer.Rating; +using BTCPayServer.Security; using BTCPayServer.Security.Greenfield; using BTCPayServer.Services; using BTCPayServer.Services.Invoices; @@ -25,6 +27,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.CodeAnalysis.CSharp.Syntax; using NBitcoin; +using NBitpayClient; using Newtonsoft.Json.Linq; using CreateInvoiceRequest = BTCPayServer.Client.Models.CreateInvoiceRequest; using InvoiceData = BTCPayServer.Client.Models.InvoiceData; @@ -96,11 +99,7 @@ public async Task GetInvoices(string storeId, [FromQuery] string[ [FromQuery] int? take = null ) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return StoreNotFound(); - } + var store = HttpContext.GetStoreData()!; if (startDate is DateTimeOffset s && endDate is DateTimeOffset e && s > e) @@ -133,17 +132,9 @@ await _invoiceRepository.GetInvoices(new InvoiceQuery() [HttpGet("~/api/v1/stores/{storeId}/invoices/{invoiceId}")] public async Task GetInvoice(string storeId, string invoiceId) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice?.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } return Ok(ToModel(invoice)); } @@ -153,16 +144,9 @@ public async Task GetInvoice(string storeId, string invoiceId) [HttpDelete("~/api/v1/stores/{storeId}/invoices/{invoiceId}")] public async Task ArchiveInvoice(string storeId, string invoiceId) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice?.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } await _invoiceRepository.ToggleInvoiceArchival(invoiceId, true, storeId); return Ok(); } @@ -172,19 +156,10 @@ public async Task ArchiveInvoice(string storeId, string invoiceId [HttpPut("~/api/v1/stores/{storeId}/invoices/{invoiceId}")] public async Task UpdateInvoice(string storeId, string invoiceId, UpdateInvoiceRequest request) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var result = await _invoiceRepository.UpdateInvoiceMetadata(invoiceId, storeId, request.Metadata); - if (result != null) - { - return Ok(ToModel(result)); - } - - return InvoiceNotFound(); + if (!BelongsToThisStore(result)) + return InvoiceNotFound(); + return Ok(ToModel(result)); } [Authorize(Policy = Policies.CanCreateInvoice, @@ -192,12 +167,7 @@ public async Task UpdateInvoice(string storeId, string invoiceId, [HttpPost("~/api/v1/stores/{storeId}/invoices")] public async Task CreateInvoice(string storeId, CreateInvoiceRequest request) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return StoreNotFound(); - } - + var store = HttpContext.GetStoreData()!; if (request.Amount < 0.0m) { ModelState.AddModelError(nameof(request.Amount), "The amount should be 0 or more."); @@ -271,17 +241,9 @@ request.Checkout.PaymentMethods[i] is not { } pm || public async Task MarkInvoiceStatus(string storeId, string invoiceId, MarkInvoiceStatusRequest request) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } if (!await _invoiceRepository.MarkInvoiceStatus(invoice.Id, request.Status)) { @@ -300,17 +262,9 @@ public async Task MarkInvoiceStatus(string storeId, string invoic [HttpPost("~/api/v1/stores/{storeId}/invoices/{invoiceId}/unarchive")] public async Task UnarchiveInvoice(string storeId, string invoiceId) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } if (!invoice.Archived) { @@ -328,21 +282,23 @@ public async Task UnarchiveInvoice(string storeId, string invoice [Authorize(Policy = Policies.CanViewInvoices, AuthenticationSchemes = AuthenticationSchemes.Greenfield)] [HttpGet("~/api/v1/stores/{storeId}/invoices/{invoiceId}/payment-methods")] - public async Task GetInvoicePaymentMethods(string storeId, string invoiceId, bool onlyAccountedPayments = true) + public async Task GetInvoicePaymentMethods(string storeId, string invoiceId, bool onlyAccountedPayments = true, bool includeSensitive = false) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice?.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } - return Ok(ToPaymentMethodModels(invoice, onlyAccountedPayments)); + if (includeSensitive && !await _authorizationService.CanModifyStore(User)) + return this.CreateAPIPermissionError(Policies.CanModifyStoreSettings); + + return Ok(ToPaymentMethodModels(invoice, onlyAccountedPayments, includeSensitive)); + } + + bool BelongsToThisStore([NotNullWhen(true)] InvoiceEntity invoice) => BelongsToThisStore(invoice, out _); + private bool BelongsToThisStore([NotNullWhen(true)] InvoiceEntity invoice, [MaybeNullWhen(false)] out Data.StoreData store) + { + store = this.HttpContext.GetStoreData(); + return invoice?.StoreId is not null && store.Id == invoice.StoreId; } [Authorize(Policy = Policies.CanViewInvoices, @@ -350,17 +306,9 @@ public async Task GetInvoicePaymentMethods(string storeId, string [HttpPost("~/api/v1/stores/{storeId}/invoices/{invoiceId}/payment-methods/{paymentMethod}/activate")] public async Task ActivateInvoicePaymentMethod(string storeId, string invoiceId, string paymentMethod) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return InvoiceNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice?.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice)) return InvoiceNotFound(); - } if (PaymentMethodId.TryParse(paymentMethod, out var paymentMethodId)) { @@ -381,22 +329,9 @@ public async Task RefundInvoice( CancellationToken cancellationToken = default ) { - var store = HttpContext.GetStoreData(); - if (store == null) - { - return StoreNotFound(); - } - var invoice = await _invoiceRepository.GetInvoice(invoiceId, true); - if (invoice == null) - { - return InvoiceNotFound(); - } - - if (invoice.StoreId != store.Id) - { + if (!BelongsToThisStore(invoice, out var store)) return InvoiceNotFound(); - } if (!invoice.GetInvoiceState().CanRefund()) { return this.CreateAPIError("non-refundable", "Cannot refund this invoice"); @@ -588,12 +523,8 @@ private IActionResult InvoiceNotFound() { return this.CreateAPIError(404, "invoice-not-found", "The invoice was not found"); } - private IActionResult StoreNotFound() - { - return this.CreateAPIError(404, "store-not-found", "The store was not found"); - } - private InvoicePaymentMethodDataModel[] ToPaymentMethodModels(InvoiceEntity entity, bool includeAccountedPaymentOnly) + private InvoicePaymentMethodDataModel[] ToPaymentMethodModels(InvoiceEntity entity, bool includeAccountedPaymentOnly, bool includeSensitive) { return entity.GetPaymentPrompts().Select( prompt => @@ -606,7 +537,12 @@ private InvoicePaymentMethodDataModel[] ToPaymentMethodModels(InvoiceEntity enti var details = prompt.Details; if (handler is not null && prompt.Activated) - details = JToken.FromObject(handler.ParsePaymentPromptDetails(details), handler.Serializer.ForAPI()); + { + var detailsObj = handler.ParsePaymentPromptDetails(details); + if (!includeSensitive) + handler.StripDetailsForNonOwner(detailsObj); + details = JToken.FromObject(detailsObj, handler.Serializer.ForAPI()); + } return new InvoicePaymentMethodDataModel { Activated = prompt.Activated, @@ -621,7 +557,7 @@ private InvoicePaymentMethodDataModel[] ToPaymentMethodModels(InvoiceEntity enti PaymentMethodFee = accounting?.PaymentMethodFee ?? 0m, PaymentLink = (prompt.Activated ? paymentLinkExtension?.GetPaymentLink(prompt, Url) : null) ?? string.Empty, Payments = payments.Select(paymentEntity => ToPaymentModel(entity, paymentEntity)).ToList(), - AdditionalData = prompt.Details + AdditionalData = details }; }).ToArray(); } diff --git a/BTCPayServer/Controllers/GreenField/GreenfieldStorePaymentMethodsController.cs b/BTCPayServer/Controllers/GreenField/GreenfieldStorePaymentMethodsController.cs index 41cf051d0b..cc02455d05 100644 --- a/BTCPayServer/Controllers/GreenField/GreenfieldStorePaymentMethodsController.cs +++ b/BTCPayServer/Controllers/GreenField/GreenfieldStorePaymentMethodsController.cs @@ -145,9 +145,7 @@ public async Task GetStorePaymentMethods( if (includeConfig is true) { - var canModifyStore = (await _authorizationService.AuthorizeAsync(User, null, - new PolicyRequirement(Policies.CanModifyStoreSettings))).Succeeded; - if (!canModifyStore) + if (!await _authorizationService.CanModifyStore(User)) return this.CreateAPIPermissionError(Policies.CanModifyStoreSettings); } diff --git a/BTCPayServer/Controllers/GreenField/LocalBTCPayServerClient.cs b/BTCPayServer/Controllers/GreenField/LocalBTCPayServerClient.cs index 1babe8558d..b647eda4e6 100644 --- a/BTCPayServer/Controllers/GreenField/LocalBTCPayServerClient.cs +++ b/BTCPayServer/Controllers/GreenField/LocalBTCPayServerClient.cs @@ -831,10 +831,12 @@ public override async Task GetInvoice(string storeId, string invoic } public override async Task GetInvoicePaymentMethods(string storeId, - string invoiceId, CancellationToken token = default) + string invoiceId, + bool onlyAccountedPayments = true, bool includeSensitive = false, + CancellationToken token = default) { return GetFromActionResult( - await GetController().GetInvoicePaymentMethods(storeId, invoiceId)); + await GetController().GetInvoicePaymentMethods(storeId, invoiceId, onlyAccountedPayments, includeSensitive)); } public override async Task ArchiveInvoice(string storeId, string invoiceId, CancellationToken token = default) diff --git a/BTCPayServer/Extensions/AuthorizationExtensions.cs b/BTCPayServer/Extensions/AuthorizationExtensions.cs index da1bc4f7b9..b3ac9108ed 100644 --- a/BTCPayServer/Extensions/AuthorizationExtensions.cs +++ b/BTCPayServer/Extensions/AuthorizationExtensions.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using BTCPayServer.Abstractions.Constants; using BTCPayServer.Client; +using BTCPayServer.Security; using BTCPayServer.Security.Bitpay; using BTCPayServer.Security.Greenfield; using BTCPayServer.Services; @@ -12,6 +13,11 @@ namespace BTCPayServer { public static class AuthorizationExtensions { + public static async Task CanModifyStore(this IAuthorizationService authorizationService, ClaimsPrincipal user) + { + return (await authorizationService.AuthorizeAsync(user, null, + new PolicyRequirement(Policies.CanModifyStoreSettings))).Succeeded; + } public static async Task<(bool HotWallet, bool RPCImport)> CanUseHotWallet( this IAuthorizationService authorizationService, PoliciesSettings policiesSettings, diff --git a/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs b/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs index 10511b3305..de4a1a2270 100644 --- a/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs +++ b/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs @@ -89,7 +89,10 @@ object IPaymentMethodHandler.ParsePaymentMethodConfig(JToken config) { return ParsePaymentMethodConfig(config); } - + public void StripDetailsForNonOwner(object details) + { + ((BitcoinPaymentPromptDetails)details).AccountDerivation = null; + } public async Task AfterSavingInvoice(PaymentMethodContext paymentMethodContext) { var paymentPrompt = paymentMethodContext.Prompt; diff --git a/BTCPayServer/Payments/Bitcoin/BitcoinPaymentPromptDetails.cs b/BTCPayServer/Payments/Bitcoin/BitcoinPaymentPromptDetails.cs index 18f5a1c956..8bf9b2ca2d 100644 --- a/BTCPayServer/Payments/Bitcoin/BitcoinPaymentPromptDetails.cs +++ b/BTCPayServer/Payments/Bitcoin/BitcoinPaymentPromptDetails.cs @@ -14,6 +14,9 @@ public class BitcoinPaymentPromptDetails [JsonConverter(typeof(StringEnumConverter))] public NetworkFeeMode FeeMode { get; set; } + /// + /// The fee rate charged to the user as `PaymentMethodFee`. + /// [JsonConverter(typeof(NBitcoin.JsonConverters.FeeRateJsonConverter))] public FeeRate PaymentMethodFeeRate { @@ -21,6 +24,10 @@ public FeeRate PaymentMethodFeeRate set; } public bool PayjoinEnabled { get; set; } + + /// + /// The recommended fee rate for this payment method. + /// [JsonConverter(typeof(NBitcoin.JsonConverters.FeeRateJsonConverter))] public FeeRate RecommendedFeeRate { get; set; } [JsonConverter(typeof(NBitcoin.JsonConverters.KeyPathJsonConverter))] diff --git a/BTCPayServer/Payments/IPaymentMethodHandler.cs b/BTCPayServer/Payments/IPaymentMethodHandler.cs index 48c454bf3a..b318e78564 100644 --- a/BTCPayServer/Payments/IPaymentMethodHandler.cs +++ b/BTCPayServer/Payments/IPaymentMethodHandler.cs @@ -67,6 +67,12 @@ public interface IPaymentMethodHandler : IHandler /// /// object ParsePaymentPromptDetails(JToken details); + /// + /// Remove properties from the details which shouldn't appear to non-store owner. + /// + /// Prompt details + void StripDetailsForNonOwner(object details) { } + /// /// Parse the configuration of the payment method in the store /// diff --git a/BTCPayServer/wwwroot/swagger/v1/swagger.template.invoices.json b/BTCPayServer/wwwroot/swagger/v1/swagger.template.invoices.json index 2463c00034..2f041fce66 100644 --- a/BTCPayServer/wwwroot/swagger/v1/swagger.template.invoices.json +++ b/BTCPayServer/wwwroot/swagger/v1/swagger.template.invoices.json @@ -413,6 +413,16 @@ "type": "boolean", "default": true } + }, + { + "name": "includeSensitive", + "in": "query", + "required": false, + "description": "If `true`, `additionalData` might include sensitive data (such as xpub). Requires the permission `btcpay.store.canmodifystoresettings`.", + "schema": { + "type": "boolean", + "default": false + } } ], "description": "View information about the specified invoice's payment methods", @@ -644,10 +654,10 @@ ] } }, - "/api/v1/stores/{storeId}/invoices/{invoiceId}/refund": { + "/api/v1/stores/{storeId}/invoices/{invoiceId}/refund": { "post": { "tags": [ - "Invoices" + "Invoices" ], "summary": "Refund invoice", "parameters": [ @@ -668,69 +678,69 @@ "schema": { "type": "string" } - } - ], - "description": "Refund invoice", - "operationId": "Invoices_Refund", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "type": "string", - "description": "Name of the pull payment (Default: 'Refund' followed by the invoice id)", - "nullable": true - }, - "description": { - "type": "string", - "description": "Description of the pull payment" - }, - "payoutMethodId": { - "$ref": "#/components/schemas/PayoutMethodId" - }, - "refundVariant": { - "type": "string", - "description": "* `RateThen`: Refund the crypto currency price, at the rate the invoice got paid.\r\n* `CurrentRate`: Refund the crypto currency price, at the current rate.\r\n*`Fiat`: Refund the invoice currency, at the rate when the refund will be sent.\r\n*`OverpaidAmount`: Refund the crypto currency amount that was overpaid.\r\n*`Custom`: Specify the amount, currency, and rate of the refund. (see `customAmount` and `customCurrency`)", - "x-enumNames": [ - "RateThen", - "CurrentRate", - "Fiat", - "Custom" - ], - "enum": [ - "RateThen", - "CurrentRate", - "OverpaidAmount", - "Fiat", - "Custom" - ] - }, - "subtractPercentage": { - "type": "string", - "format": "decimal", - "description": "Optional percentage by which to reduce the refund, e.g. as processing charge or to compensate for the mining fee.", - "example": "2.1" - }, - "customAmount": { - "type": "string", - "format": "decimal", - "description": "The amount to refund if the `refundVariant` is `Custom`.", - "example": "5.00" - }, - "customCurrency": { - "type": "string", - "description": "The currency to refund if the `refundVariant` is `Custom`", - "example": "USD" - } - } - } + } + ], + "description": "Refund invoice", + "operationId": "Invoices_Refund", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name of the pull payment (Default: 'Refund' followed by the invoice id)", + "nullable": true + }, + "description": { + "type": "string", + "description": "Description of the pull payment" + }, + "payoutMethodId": { + "$ref": "#/components/schemas/PayoutMethodId" + }, + "refundVariant": { + "type": "string", + "description": "* `RateThen`: Refund the crypto currency price, at the rate the invoice got paid.\r\n* `CurrentRate`: Refund the crypto currency price, at the current rate.\r\n*`Fiat`: Refund the invoice currency, at the rate when the refund will be sent.\r\n*`OverpaidAmount`: Refund the crypto currency amount that was overpaid.\r\n*`Custom`: Specify the amount, currency, and rate of the refund. (see `customAmount` and `customCurrency`)", + "x-enumNames": [ + "RateThen", + "CurrentRate", + "Fiat", + "Custom" + ], + "enum": [ + "RateThen", + "CurrentRate", + "OverpaidAmount", + "Fiat", + "Custom" + ] + }, + "subtractPercentage": { + "type": "string", + "format": "decimal", + "description": "Optional percentage by which to reduce the refund, e.g. as processing charge or to compensate for the mining fee.", + "example": "2.1" + }, + "customAmount": { + "type": "string", + "format": "decimal", + "description": "The amount to refund if the `refundVariant` is `Custom`.", + "example": "5.00" + }, + "customCurrency": { + "type": "string", + "description": "The currency to refund if the `refundVariant` is `Custom`", + "example": "USD" + } + } + } } } - }, + }, "responses": { "200": { "description": "Pull payment for refunding the invoice", @@ -1329,6 +1339,7 @@ "anyOf": [ { "type": "object", + "title": "*-LNURL", "description": "LNURL Pay information", "properties": { "providedComment": { @@ -1345,6 +1356,39 @@ } } }, + { + "type": "object", + "title": "*-CHAIN", + "description": "Bitcoin On-Chain payment information", + "properties": { + "keyPath": { + "type": "string", + "description": "The key path relative to the account derviation key.", + "example": "0/1" + }, + "payjoinEnabled": { + "type": "boolean", + "description": "If the payjoin feature is enabled for this payment method." + }, + "accountDerivation": { + "type": "string", + "description": "The derivation scheme used to derive addresses (null if `includeSensitive` is `false`)", + "example": "xpub6DVMcQAQCtGbNDTEjQGtR1GRoTKw7AzP6bVivX4gFnewcnRk1r1tbczpfsaYjKKVrmtyiwYqAEnALYzZ8yoTArVsKfZekmwLFqQp4MRgPhy" + }, + "recommendedFeeRate": { + "type": "string", + "format": "decimal", + "description": "The recommended fee rate for this payment method.", + "example": "4.107" + }, + "paymentMethodFeeRate": { + "type": "string", + "format": "decimal", + "description": "The fee rate charged to the user as `PaymentMethodFee`.", + "example": "3.975" + } + } + }, { "type": "object", "description": "No additional information" From 83fa8cbf0f5fa4580f9e4b0b4bb31fe76da2e9de Mon Sep 17 00:00:00 2001 From: Chukwuleta Tobechi <47084273+TChukwuleta@users.noreply.github.com> Date: Fri, 27 Sep 2024 07:28:55 +0100 Subject: [PATCH 25/26] prevent app creation without wallet creation (#6255) * prevent app creation without wallet creation * resolve test failures * resolve selenium test --- BTCPayServer.Tests/CrowdfundTests.cs | 2 ++ BTCPayServer.Tests/SeleniumTests.cs | 4 +++- BTCPayServer.Tests/UnitTest1.cs | 2 ++ BTCPayServer/Controllers/UIAppsController.cs | 17 +++++++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/BTCPayServer.Tests/CrowdfundTests.cs b/BTCPayServer.Tests/CrowdfundTests.cs index cfedd8b7c8..31bdb9393c 100644 --- a/BTCPayServer.Tests/CrowdfundTests.cs +++ b/BTCPayServer.Tests/CrowdfundTests.cs @@ -39,6 +39,8 @@ public async Task CanCreateAndDeleteCrowdfundApp() await user.GrantAccessAsync(); var user2 = tester.NewAccount(); await user2.GrantAccessAsync(); + await user.RegisterDerivationSchemeAsync("BTC"); + await user2.RegisterDerivationSchemeAsync("BTC"); var apps = user.GetController(); var apps2 = user2.GetController(); var crowdfund = user.GetController(); diff --git a/BTCPayServer.Tests/SeleniumTests.cs b/BTCPayServer.Tests/SeleniumTests.cs index 53c008d52b..cdf579e9f7 100644 --- a/BTCPayServer.Tests/SeleniumTests.cs +++ b/BTCPayServer.Tests/SeleniumTests.cs @@ -1238,6 +1238,7 @@ public async Task CanCreateAppPoS() await s.StartAsync(); var userId = s.RegisterNewUser(true); s.CreateNewStore(); + s.GenerateWallet(); (_, string appId) = s.CreateApp("PointOfSale"); s.Driver.FindElement(By.Id("Title")).Clear(); s.Driver.FindElement(By.Id("Title")).SendKeys("Tea shop"); @@ -1249,7 +1250,8 @@ public async Task CanCreateAppPoS() s.Driver.FindElement(By.Id("CodeTabButton")).Click(); var template = s.Driver.FindElement(By.Id("TemplateConfig")).GetAttribute("value"); Assert.Contains("\"buyButtonText\": \"Take my money\"", template); - Assert.Matches("\"categories\": \\[\n\\s+\"Drinks\"\n\\s+\\]", template); + Assert.Matches("\"categories\": \\[\r?\n\\s*\"Drinks\"\\s*\\]", template); + s.ClickPagePrimary(); Assert.Contains("App updated", s.FindAlertMessage().Text); diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index f350dcde4b..812cca065e 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -1844,6 +1844,8 @@ public async Task CanCreateAndDeleteApps() await user.GrantAccessAsync(); var user2 = tester.NewAccount(); await user2.GrantAccessAsync(); + await user.RegisterDerivationSchemeAsync("BTC"); + await user2.RegisterDerivationSchemeAsync("BTC"); var stores = user.GetController(); var apps = user.GetController(); var apps2 = user2.GetController(); diff --git a/BTCPayServer/Controllers/UIAppsController.cs b/BTCPayServer/Controllers/UIAppsController.cs index f44d0acbd5..d044df0593 100644 --- a/BTCPayServer/Controllers/UIAppsController.cs +++ b/BTCPayServer/Controllers/UIAppsController.cs @@ -24,12 +24,14 @@ public partial class UIAppsController : Controller { public UIAppsController( UserManager userManager, + BTCPayNetworkProvider networkProvider, StoreRepository storeRepository, IFileService fileService, AppService appService, IHtmlHelper html) { _userManager = userManager; + _networkProvider = networkProvider; _storeRepository = storeRepository; _fileService = fileService; _appService = appService; @@ -37,6 +39,7 @@ public UIAppsController( } private readonly UserManager _userManager; + private readonly BTCPayNetworkProvider _networkProvider; private readonly StoreRepository _storeRepository; private readonly IFileService _fileService; private readonly AppService _appService; @@ -133,6 +136,20 @@ public IActionResult CreateApp(string storeId, string appType = null) public async Task CreateApp(string storeId, CreateAppViewModel vm) { var store = GetCurrentStore(); + if (store == null) + { + return NotFound(); + } + if (!store.AnyPaymentMethodAvailable()) + { + TempData.SetStatusMessageModel(new StatusMessageModel + { + Severity = StatusMessageModel.StatusSeverity.Error, + Html = $"To create a {vm.AppType} app, you need to set up a wallet first", + AllowDismiss = false + }); + return View(vm); + } vm.StoreId = store.Id; var type = _appService.GetAppType(vm.AppType ?? vm.SelectedAppType); if (type is null) From 6d284b4124ba5a220cd31befd58fc8816c2a0a3e Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 27 Sep 2024 15:48:16 +0900 Subject: [PATCH 26/26] Give time for pollers to detect payments after server restart --- BTCPayServer/HostedServices/InvoiceWatcher.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/BTCPayServer/HostedServices/InvoiceWatcher.cs b/BTCPayServer/HostedServices/InvoiceWatcher.cs index cc094a534a..fceda6d82b 100644 --- a/BTCPayServer/HostedServices/InvoiceWatcher.cs +++ b/BTCPayServer/HostedServices/InvoiceWatcher.cs @@ -192,13 +192,20 @@ private void Watch(string invoiceId) } } - private async Task Wait(string invoiceId) => await Wait(await _invoiceRepository.GetInvoice(invoiceId)); - private async Task Wait(InvoiceEntity invoice) + private async Task Wait(string invoiceId, bool startup) => await Wait(await _invoiceRepository.GetInvoice(invoiceId), startup); + private async Task Wait(InvoiceEntity invoice, bool startup) { + var startupOffset = TimeSpan.Zero; + + // This give some times for the pollers in the listeners to catch payments which happened + // while the server was down. + if (startup) + startupOffset += TimeSpan.FromMinutes(2.0); + try { // add 1 second to ensure watch won't trigger moments before invoice expires - var delay = invoice.ExpirationTime.AddSeconds(1) - DateTimeOffset.UtcNow; + var delay = (invoice.ExpirationTime.AddSeconds(1) + startupOffset) - DateTimeOffset.UtcNow; if (delay > TimeSpan.Zero) { await Task.Delay(delay, _Cts.Token); @@ -243,7 +250,7 @@ await _notificationSender.SendNotification(new StoreScope(b.Invoice.StoreId), if (b.Name == InvoiceEvent.Created) { Watch(b.Invoice.Id); - _ = Wait(b.Invoice.Id); + _ = Wait(b.Invoice.Id, false); } if (b.Name == InvoiceEvent.ReceivedPayment) @@ -257,7 +264,7 @@ await _notificationSender.SendNotification(new StoreScope(b.Invoice.StoreId), private async Task WaitPendingInvoices() { await Task.WhenAll((await GetPendingInvoices(_Cts.Token)) - .Select(i => Wait(i)).ToArray()); + .Select(i => Wait(i, true)).ToArray()); } private async Task GetPendingInvoices(CancellationToken cancellationToken)