From 7569ca67d722282e15de76f8b01ffc60d3881ace Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 11 Mar 2024 11:17:09 +0500 Subject: [PATCH 01/14] Update DotSettings (rider migration) --- MarginTrading.sln.DotSettings | 1 + 1 file changed, 1 insertion(+) diff --git a/MarginTrading.sln.DotSettings b/MarginTrading.sln.DotSettings index 5e9904edd..b99bda025 100644 --- a/MarginTrading.sln.DotSettings +++ b/MarginTrading.sln.DotSettings @@ -1,4 +1,5 @@  + True True True True From 519d10670f484a437d5f8302cc4d653aab8d65a2 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Fri, 15 Mar 2024 15:40:27 +0500 Subject: [PATCH 02/14] feat(LT-5012): convert draft snapshot to final during normal EOD --- .../Settings/BookKeeperServiceClient.cs | 18 +++++++++ .../Infrastructure/SnapshotService.cs | 7 +++- .../MarginTrading.Backend.Services.csproj | 2 +- .../Settings/MtBackendSettings.cs | 2 + .../Workflow/EodCommandsHandler.cs | 39 +++++++++++++++++-- .../Modules/ExternalServicesModule.cs | 31 +++++++++++++++ 6 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs diff --git a/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs new file mode 100644 index 000000000..f6989171e --- /dev/null +++ b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs @@ -0,0 +1,18 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using JetBrains.Annotations; +using Lykke.SettingsReader.Attributes; + +namespace MarginTrading.Backend.Core.Settings +{ + [UsedImplicitly] + public class BookKeeperServiceClient + { + [HttpCheck("/api/isalive")] + public string ServiceUrl { get; set; } + + [Optional] + public string ApiKey { get; set; } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs index 4605456a3..1d5483935 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs @@ -162,6 +162,11 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho var accountsJson = accountStats .Select(a => a.ConvertToSnapshotContract(accountsInLiquidation.Contains(a), status)) .ToJson(); + + // timestamp will be used as an eod border + // setting it as close as possible to accountStats retrieval + var timestamp = _dateService.Now(); + await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), $"Preparing data... {accountStats.Count} accounts prepared."); @@ -183,7 +188,7 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho var snapshot = new TradingEngineSnapshot( tradingDay, correlationId, - _dateService.Now(), + timestamp, ordersJson: ordersJson, positionsJson: positionsJson, accountsJson: accountsJson, diff --git a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj index 7aab54792..84506d186 100644 --- a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj +++ b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs b/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs index e152511da..95ab0d628 100644 --- a/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs +++ b/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs @@ -31,6 +31,8 @@ public class MtBackendSettings public ExchangeConnectorServiceClient MtStpExchangeConnectorClient { get; set; } public SettingsServiceClient SettingsServiceClient { get; set; } + + public BookKeeperServiceClient BookKeeperServiceClient { get; set; } public AccountsManagementServiceClient AccountsManagementServiceClient { get; set; } diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index d137dd412..d98854d53 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -2,12 +2,17 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using BookKeeper.Client; +using BookKeeper.Client.Responses.Eod; using BookKeeper.Client.Workflow.Commands; using BookKeeper.Client.Workflow.Events; using Common.Log; using JetBrains.Annotations; using Lykke.Cqrs; +using MarginTrading.Backend.Contracts.Prices; using MarginTrading.Backend.Core.Services; using MarginTrading.Common.Services; @@ -16,12 +21,18 @@ namespace MarginTrading.Backend.Services.Workflow [UsedImplicitly] public class EodCommandsHandler { + private readonly IQuotesApi _quotesApi; private readonly ISnapshotService _snapshotService; private readonly IDateService _dateService; private readonly ILog _log; - public EodCommandsHandler(ISnapshotService snapshotService, IDateService dateService, ILog log) + public EodCommandsHandler( + IQuotesApi quotesApi, + ISnapshotService snapshotService, + IDateService dateService, + ILog log) { + _quotesApi = quotesApi; _snapshotService = snapshotService; _dateService = dateService; _log = log; @@ -30,10 +41,14 @@ public EodCommandsHandler(ISnapshotService snapshotService, IDateService dateSer [UsedImplicitly] private async Task Handle(CreateSnapshotCommand command, IEventPublisher publisher) { - //deduplication is inside _snapshotService.MakeTradingDataSnapshot + //deduplication is inside _snapshotService try { - await _snapshotService.MakeTradingDataSnapshot(command.TradingDay, command.OperationId); + var quotes = await _quotesApi.GetCfdQuotes(command.TradingDay); + + await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, + MapQuotes(quotes.EodMarketData.Underlyings), + MapFxRates(quotes.EodMarketData.Forex)); publisher.PublishEvent(new SnapshotCreatedEvent { @@ -54,5 +69,23 @@ await _log.WriteErrorAsync(nameof(EodCommandsHandler), nameof(CreateSnapshotComm }); } } + + private IEnumerable MapQuotes(IEnumerable bestPrices) + { + return bestPrices.Select(x => new ClosingAssetPrice() + { + ClosePrice = x.Ask, // equal to bid + AssetId = x.Id, + }); + } + + private IEnumerable MapFxRates(IEnumerable bestPrices) + { + return bestPrices.Select(x => new ClosingFxRate() + { + ClosePrice = x.Ask, // equal to bid + AssetId = x.Id, + }); + } } } \ No newline at end of file diff --git a/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs b/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs index d76d5f2e4..33cd82429 100644 --- a/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs +++ b/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs @@ -3,6 +3,7 @@ using System; using Autofac; +using BookKeeper.Client; using Common.Log; using Lykke.HttpClientGenerator; using Lykke.HttpClientGenerator.Retries; @@ -186,6 +187,16 @@ protected override void Load(ContainerBuilder builder) .As().SingleInstance(); #endregion OrderBook Service + + #region BookKeeper + + builder + .Register(ctx => BuildBookKeeperClientGenerator(ctx) + .Generate()) + .As() + .SingleInstance(); + + #endregion } private HttpClientGenerator BuildAccountManagementClientGenerator(IComponentContext ctx) @@ -222,5 +233,25 @@ private HttpClientGenerator BuildSettingsClientGenerator(IComponentContext ctx) return settingsClientGeneratorBuilder.Create(); } + + private HttpClientGenerator BuildBookKeeperClientGenerator(IComponentContext ctx) + { + var bookkeeperSettings = _settings.CurrentValue.BookKeeperServiceClient; + + var bookkeeperClientGeneratorBuilder = HttpClientGenerator + .BuildForUrl(bookkeeperSettings.ServiceUrl) + .WithAdditionalDelegatingHandler(ctx.Resolve()) + .WithServiceName( + $"BookKeeper [{bookkeeperSettings.ServiceUrl}]") + .WithRetriesStrategy(new LinearRetryStrategy(TimeSpan.FromMilliseconds(300), 3)); + + if (!string.IsNullOrWhiteSpace(bookkeeperSettings.ApiKey)) + { + bookkeeperClientGeneratorBuilder = bookkeeperClientGeneratorBuilder + .WithApiKey(bookkeeperSettings.ApiKey); + } + + return bookkeeperClientGeneratorBuilder.Create(); + } } } \ No newline at end of file From d678809b35296743dc227a893c988c5a021410e6 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 11:06:20 +0500 Subject: [PATCH 03/14] feat(LT-5012): add snapshot monitoring --- .../Services/ISnapshotMonitor.cs | 15 ++++ .../Settings/MarginTradingSettings.cs | 3 + .../Settings/SnapshotMonitorSettings.cs | 21 ++++++ .../ScheduleSettingsCacheService.cs | 21 +++++- .../Infrastructure/SnapshotMonitor.cs | 64 ++++++++++++++++ .../Infrastructure/SnapshotService.cs | 8 +- .../Modules/ManagersModule.cs | 4 + .../SnapshotMonitorService.cs | 73 +++++++++++++++++++ .../Modules/BackendSettingsModule.cs | 1 + src/MarginTrading.Backend/Startup.cs | 2 + 10 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs create mode 100644 src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs create mode 100644 src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs create mode 100644 src/MarginTrading.Backend.Services/SnapshotMonitorService.cs diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs new file mode 100644 index 000000000..25c566b3f --- /dev/null +++ b/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs @@ -0,0 +1,15 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Core.Services +{ + public interface ISnapshotMonitor + { + void SnapshotRequested(DateTime tradingDay); + void SnapshotInProgress(); + void SnapshotFinished(); + bool ShouldRetrySnapshot(out DateTime tradingDay); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs index a6c054ba3..c76f3a942 100644 --- a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs +++ b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs @@ -5,6 +5,7 @@ using JetBrains.Annotations; using Lykke.Common.Chaos; using Lykke.SettingsReader.Attributes; +using MarginTrading.Backend.Core.Services; using MarginTrading.Common.RabbitMq; using MarginTrading.Common.Settings; @@ -129,5 +130,7 @@ public class MarginTradingSettings // todo: probably should be moved turned in to a feature flag [Optional] public bool PerformanceTrackerEnabled { get; set; } = false; + + [Optional] public SnapshotMonitorSettings SnapshotMonitorSettings { get; set; } } } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs new file mode 100644 index 000000000..5e8723e47 --- /dev/null +++ b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs @@ -0,0 +1,21 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Core.Settings +{ + public class SnapshotMonitorSettings + { + /// + /// Defines the interval between consecutive checks performed by the SnapshotMonitoring service. + /// + public TimeSpan MonitoringDelay { get; set; } + + /// + /// If snapshot is not created after a specified amount of time, SnapshotMonitorService will retry this operation + /// + + public TimeSpan SnapshotCreationTimeout { get; set; } = TimeSpan.FromMinutes(5); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs index fea501868..2554c4067 100644 --- a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs +++ b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs @@ -19,6 +19,8 @@ using MarginTrading.Common.Services; using MarginTrading.AssetService.Contracts; using MarginTrading.AssetService.Contracts.Scheduling; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Services.Extensions; using Microsoft.FeatureManagement; using MoreLinq; @@ -49,6 +51,7 @@ public class ScheduleSettingsCacheService : IScheduleSettingsCacheService private readonly ReaderWriterLockSlim _readerWriterLockSlim = new ReaderWriterLockSlim(); private readonly IFeatureManager _featureManager; + private readonly ISnapshotMonitor _snapshotMonitor; public ScheduleSettingsCacheService( ICqrsSender cqrsSender, @@ -57,7 +60,8 @@ public ScheduleSettingsCacheService( IDateService dateService, ILog log, OvernightMarginSettings overnightMarginSettings, - IFeatureManager featureManager) + IFeatureManager featureManager, + ISnapshotMonitor snapshotMonitor) { _cqrsSender = cqrsSender; _scheduleSettingsApi = scheduleSettingsApi; @@ -66,6 +70,7 @@ public ScheduleSettingsCacheService( _log = log; _overnightMarginSettings = overnightMarginSettings; _featureManager = featureManager; + _snapshotMonitor = snapshotMonitor; } public async Task UpdateAllSettingsAsync() @@ -185,12 +190,20 @@ private void HandleMarketStateChangesUnsafe(DateTime currentTime, string[] marke .Where(x => marketIds.IsNullOrEmpty() || marketIds.Contains(x.Key))) { var newState = scheduleSettings.GetMarketState(marketId, currentTime); - _cqrsSender.PublishEvent(new MarketStateChangedEvent + + var now = _dateService.Now(); + var ev = new MarketStateChangedEvent { Id = marketId, IsEnabled = newState.IsEnabled, - EventTimestamp = _dateService.Now(), - }); + EventTimestamp = now, + }; + + if (ev.IsPlatformClosureEvent()) + { + _snapshotMonitor.SnapshotRequested(now.Date); + } + _cqrsSender.PublishEvent(ev); _marketStates[marketId] = newState; } diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs new file mode 100644 index 000000000..9029aae97 --- /dev/null +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs @@ -0,0 +1,64 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Core.Settings; + +namespace MarginTrading.Backend.Services.Infrastructure +{ + /// + /// This class decouples draft snapshot creation from rabbit mq + /// Normal flow: MarketStateChangedEvent generated => handled in PlatformClosureProjection => snapshot saved + /// Degraded flow: MarketStateChangedEvent generated, snapshot requested => event not received in PlatformClosureProjection => SnapshotMonitorService retries snapshot creation after a timeout + /// + public class SnapshotMonitor : ISnapshotMonitor + { + private readonly SnapshotMonitorSettings _settings; + private DraftSnapshotState _state = DraftSnapshotState.None; + private DateTime _timestamp; + private DateTime _tradingDay; + + public SnapshotMonitor(SnapshotMonitorSettings settings) + { + _settings = settings; + } + + // TODO: concurrency / thread safety? + public void SnapshotRequested(DateTime tradingDay) + { + _state = DraftSnapshotState.Requested; + _timestamp = DateTime.UtcNow; + _tradingDay = tradingDay; + } + + public void SnapshotInProgress() + { + _state = DraftSnapshotState.InProgress; + } + + public void SnapshotFinished() + { + _state = DraftSnapshotState.None; + _timestamp = default; + _tradingDay = default; + } + + public bool ShouldRetrySnapshot(out DateTime tradingDay) + { + var shouldRetry = _state == DraftSnapshotState.Requested && + _timestamp.Add(_settings.SnapshotCreationTimeout) <= DateTime.UtcNow; + + tradingDay = shouldRetry ? _tradingDay : default; + + return shouldRetry; + } + + private enum DraftSnapshotState + { + None, + Requested, + InProgress, + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs index 1d5483935..0e0100030 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs @@ -36,6 +36,7 @@ public class SnapshotService : ISnapshotService private readonly IMarginTradingBlobRepository _blobRepository; private readonly ILog _log; private readonly IFinalSnapshotCalculator _finalSnapshotCalculator; + private readonly ISnapshotMonitor _snapshotMonitor; private readonly MarginTradingSettings _settings; private static readonly SemaphoreSlim Lock = new SemaphoreSlim(1, 1); @@ -54,6 +55,7 @@ public SnapshotService( IMarginTradingBlobRepository blobRepository, ILog log, IFinalSnapshotCalculator finalSnapshotCalculator, + ISnapshotMonitor snapshotMonitor, MarginTradingSettings settings) { _scheduleSettingsCacheService = scheduleSettingsCacheService; @@ -68,6 +70,7 @@ public SnapshotService( _blobRepository = blobRepository; _log = log; _finalSnapshotCalculator = finalSnapshotCalculator; + _snapshotMonitor = snapshotMonitor; _settings = settings; } @@ -123,6 +126,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho try { + _snapshotMonitor.SnapshotInProgress(); + var orders = _orderReader.GetAllOrders(); var ordersJson = orders.Select(o => o.ConvertToSnapshotContract(_orderReader, status)).ToJson(); await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), @@ -197,6 +202,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho status: status); await _tradingEngineSnapshotsRepository.AddAsync(snapshot); + + _snapshotMonitor.SnapshotFinished(); await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), $"Trading data snapshot was written to the storage. {msg}"); @@ -223,7 +230,6 @@ public async Task MakeTradingDataSnapshotFromDraft( try { var snapshot = await _finalSnapshotCalculator.RunAsync(fxRates, cfdQuotes, correlationId); - await _tradingEngineSnapshotsRepository.AddAsync(snapshot); } finally diff --git a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs index b71874307..9a3f59983 100644 --- a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs +++ b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs @@ -57,6 +57,10 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .SingleInstance(); + + builder.RegisterType() + .As() + .SingleInstance(); } } } diff --git a/src/MarginTrading.Backend.Services/SnapshotMonitorService.cs b/src/MarginTrading.Backend.Services/SnapshotMonitorService.cs new file mode 100644 index 000000000..c0c707c27 --- /dev/null +++ b/src/MarginTrading.Backend.Services/SnapshotMonitorService.cs @@ -0,0 +1,73 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using MarginTrading.Backend.Core.Repositories; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Core.Settings; +using MarginTrading.Backend.Core.Snapshots; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace MarginTrading.Backend.Services +{ + public class SnapshotMonitorService : BackgroundService + { + private readonly ISnapshotMonitor _snapshotMonitor; + private readonly ISnapshotService _snapshotService; + private readonly IIdentityGenerator _identityGenerator; + private readonly SnapshotMonitorSettings _settings; + private readonly ILogger _logger; + + public SnapshotMonitorService( + ISnapshotMonitor snapshotMonitor, + ISnapshotService snapshotService, + IIdentityGenerator identityGenerator, + SnapshotMonitorSettings settings, + ILogger logger) + { + _snapshotMonitor = snapshotMonitor; + _snapshotService = snapshotService; + _identityGenerator = identityGenerator; + _settings = settings; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("{ServiceName} started", nameof(SnapshotMonitorService)); + + while (!stoppingToken.IsCancellationRequested) + { + if (_snapshotMonitor.ShouldRetrySnapshot(out var tradingDay)) + { + _logger.LogWarning("{ServiceName}: Trading Snapshot Draft was requested, but timeout exceeded. Attempting to create the snapshot.", + nameof(SnapshotMonitorService)); + + try + { + await _snapshotService.MakeTradingDataSnapshot(tradingDay, + _identityGenerator.GenerateGuid(), + SnapshotStatus.Draft); + + _logger.LogInformation("{ServiceName}: Trading Snapshot Draft was created", + nameof(SnapshotMonitorService)); + } + catch (Exception ex) + { + _logger.LogCritical(ex, + "Could not create trading data snapshot for {TradingDay}. {Message}", + tradingDay, + ex.Message); + + // exception is swallowed to allow retries + } + } + + await Task.Delay(_settings.MonitoringDelay, stoppingToken); + } + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs b/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs index c86966973..f2481e0a9 100644 --- a/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs +++ b/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs @@ -27,6 +27,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterInstance(_settings.CurrentValue.RiskInformingSettings ?? new RiskInformingSettings {Data = new RiskInformingParams[0]}).SingleInstance(); builder.RegisterInstance(_settings.CurrentValue.MtBackend.OvernightMargin).SingleInstance(); + builder.RegisterInstance(_settings.CurrentValue.MtBackend.SnapshotMonitorSettings).SingleInstance(); } } } diff --git a/src/MarginTrading.Backend/Startup.cs b/src/MarginTrading.Backend/Startup.cs index 0b3964447..6a42a588f 100644 --- a/src/MarginTrading.Backend/Startup.cs +++ b/src/MarginTrading.Backend/Startup.cs @@ -129,6 +129,8 @@ public void ConfigureServices(IServiceCollection services) services.AddFeatureManagement(_mtSettingsManager.CurrentValue.MtBackend); SetupLoggers(Configuration, services, _mtSettingsManager, correlationContextAccessor); + + services.AddHostedService(); } [UsedImplicitly] From d9494a436d0ed034fd45acaf1eddea379d052654 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 11:07:21 +0500 Subject: [PATCH 04/14] fix(LT-5012): provide default value for MonitoringDelay --- .../Settings/SnapshotMonitorSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs index 5e8723e47..b0c961d07 100644 --- a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs +++ b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs @@ -10,7 +10,7 @@ public class SnapshotMonitorSettings /// /// Defines the interval between consecutive checks performed by the SnapshotMonitoring service. /// - public TimeSpan MonitoringDelay { get; set; } + public TimeSpan MonitoringDelay { get; set; } = TimeSpan.FromSeconds(30); /// /// If snapshot is not created after a specified amount of time, SnapshotMonitorService will retry this operation From 0febac10094c590e788171951fa04f257d65bcf1 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 11:15:42 +0500 Subject: [PATCH 05/14] fix(LT-5012): handle missing quotes in CreateSnapshotCommand handler --- .../Workflow/EodCommandsHandler.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index d98854d53..7427f0874 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -45,6 +45,11 @@ private async Task Handle(CreateSnapshotCommand command, IEventPublisher publish try { var quotes = await _quotesApi.GetCfdQuotes(command.TradingDay); + + if (quotes.ErrorCode != EodMarketDataErrorCodesContract.None) + { + throw new Exception($"Could not receive quotes from BookKeeper: {quotes.ErrorCode.ToString()}"); + } await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, MapQuotes(quotes.EodMarketData.Underlyings), From 910569cec555ab8e905dbdd5468bfcc9266e8449 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 11:26:29 +0500 Subject: [PATCH 06/14] fix(LT-5012): build --- tests/MarginTradingTests/AssetDayOffTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/MarginTradingTests/AssetDayOffTests.cs b/tests/MarginTradingTests/AssetDayOffTests.cs index 9b384cd12..5e8273281 100644 --- a/tests/MarginTradingTests/AssetDayOffTests.cs +++ b/tests/MarginTradingTests/AssetDayOffTests.cs @@ -328,7 +328,8 @@ private IAssetPairDayOffService ArrangeDayOffService(DateTime dateTime, dateService.Object, new EmptyLog(), new OvernightMarginSettings(), - Mock.Of()); + Mock.Of(), + new SnapshotMonitor(new SnapshotMonitorSettings())); scheduleSettingsCacheService.UpdateAllSettingsAsync().GetAwaiter().GetResult(); return new AssetPairDayOffService(scheduleSettingsCacheService); From c3af2516fdf02b402a8a2398d89df0c44cd4bd56 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 11:37:28 +0500 Subject: [PATCH 07/14] fix(LT-5012): tests --- .../Settings/MarginTradingSettings.cs | 4 ++-- tests/MarginTradingTests/BaseTests.cs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs index c76f3a942..1187e34c4 100644 --- a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs +++ b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs @@ -130,7 +130,7 @@ public class MarginTradingSettings // todo: probably should be moved turned in to a feature flag [Optional] public bool PerformanceTrackerEnabled { get; set; } = false; - - [Optional] public SnapshotMonitorSettings SnapshotMonitorSettings { get; set; } + + [Optional] public SnapshotMonitorSettings SnapshotMonitorSettings { get; set; } = new SnapshotMonitorSettings(); } } \ No newline at end of file diff --git a/tests/MarginTradingTests/BaseTests.cs b/tests/MarginTradingTests/BaseTests.cs index 3a0d0acfe..aa72a5618 100644 --- a/tests/MarginTradingTests/BaseTests.cs +++ b/tests/MarginTradingTests/BaseTests.cs @@ -80,10 +80,12 @@ private void RegisterDependenciesCore(bool mockEvents = false) }, ReportingEquivalentPricesSettings = new[] {new ReportingEquivalentPricesSettings {EquivalentAsset = "USD", LegalEntity = "LYKKETEST"}}, - OvernightMargin = overnightMarginSettings + OvernightMargin = overnightMarginSettings, + SnapshotMonitorSettings = new SnapshotMonitorSettings(), }; builder.RegisterInstance(marginSettings).SingleInstance(); + builder.RegisterInstance(marginSettings.SnapshotMonitorSettings).SingleInstance(); builder.RegisterInstance(PositionHistoryEvents).As>().SingleInstance(); builder.RegisterInstance(overnightMarginSettings).SingleInstance(); builder.RegisterInstance(Mock.Of()); From 85bcdc5b8ec18366a45b730a9ff54913256bb2bf Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 12:03:44 +0500 Subject: [PATCH 08/14] fix(LT-5012): fix refit error --- .../MarginTrading.Backend.Services.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj index 84506d186..1b161eda3 100644 --- a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj +++ b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj @@ -28,7 +28,7 @@ - + From 7f32a8a3c748ec6b522637e9a9a5e72ff6015246 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 12:11:35 +0500 Subject: [PATCH 09/14] fix(LT-5012): remove isAlive check --- .../Settings/BookKeeperServiceClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs index f6989171e..7874146c4 100644 --- a/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs +++ b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs @@ -9,7 +9,7 @@ namespace MarginTrading.Backend.Core.Settings [UsedImplicitly] public class BookKeeperServiceClient { - [HttpCheck("/api/isalive")] + // isAlive check leads to a deadlock public string ServiceUrl { get; set; } [Optional] From c44c63afa7aa06210770039b6d1ad8b82822a05b Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 13:23:39 +0500 Subject: [PATCH 10/14] fix(LT-5012): init draftSnapshotKeeper --- .../Workflow/EodCommandsHandler.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index 7427f0874..91ce3f500 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -15,6 +15,7 @@ using MarginTrading.Backend.Contracts.Prices; using MarginTrading.Backend.Core.Services; using MarginTrading.Common.Services; +using Microsoft.Extensions.DependencyInjection; namespace MarginTrading.Backend.Services.Workflow { @@ -24,17 +25,20 @@ public class EodCommandsHandler private readonly IQuotesApi _quotesApi; private readonly ISnapshotService _snapshotService; private readonly IDateService _dateService; + private readonly IServiceScopeFactory _serviceScopeFactory; private readonly ILog _log; public EodCommandsHandler( IQuotesApi quotesApi, ISnapshotService snapshotService, - IDateService dateService, + IDateService dateService, + IServiceScopeFactory serviceScopeFactory, ILog log) { _quotesApi = quotesApi; _snapshotService = snapshotService; _dateService = dateService; + _serviceScopeFactory = serviceScopeFactory; _log = log; } @@ -50,6 +54,10 @@ private async Task Handle(CreateSnapshotCommand command, IEventPublisher publish { throw new Exception($"Could not receive quotes from BookKeeper: {quotes.ErrorCode.ToString()}"); } + + using var scope = _serviceScopeFactory.CreateScope(); + var draftSnapshotKeeper = scope.ServiceProvider.GetService(); + draftSnapshotKeeper.Init(command.TradingDay); await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, MapQuotes(quotes.EodMarketData.Underlyings), From 2250b1ff533942befdd4df7e1dfb78034f130997 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Mon, 18 Mar 2024 14:40:16 +0500 Subject: [PATCH 11/14] fix(LT-5102): another attempt at IDraftSnapshotKeeper --- .../Workflow/EodCommandsHandler.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index 91ce3f500..c3c255cfa 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -25,20 +25,20 @@ public class EodCommandsHandler private readonly IQuotesApi _quotesApi; private readonly ISnapshotService _snapshotService; private readonly IDateService _dateService; - private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly IDraftSnapshotKeeper _draftSnapshotKeeper; private readonly ILog _log; public EodCommandsHandler( IQuotesApi quotesApi, ISnapshotService snapshotService, IDateService dateService, - IServiceScopeFactory serviceScopeFactory, + IDraftSnapshotKeeper draftSnapshotKeeper, ILog log) { _quotesApi = quotesApi; _snapshotService = snapshotService; _dateService = dateService; - _serviceScopeFactory = serviceScopeFactory; + _draftSnapshotKeeper = draftSnapshotKeeper; _log = log; } @@ -55,9 +55,7 @@ private async Task Handle(CreateSnapshotCommand command, IEventPublisher publish throw new Exception($"Could not receive quotes from BookKeeper: {quotes.ErrorCode.ToString()}"); } - using var scope = _serviceScopeFactory.CreateScope(); - var draftSnapshotKeeper = scope.ServiceProvider.GetService(); - draftSnapshotKeeper.Init(command.TradingDay); + _draftSnapshotKeeper.Init(command.TradingDay); await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, MapQuotes(quotes.EodMarketData.Underlyings), From df49d41df2af14be0509aa4a50c4c437bc2677aa Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Tue, 19 Mar 2024 13:34:04 +0500 Subject: [PATCH 12/14] fix(LT-5012): add DraftSnapshotKeeperFactory --- .../Services/ISnapshotService.cs | 27 +------------ .../DraftSnapshotKeeperFactory.cs | 25 ++++++++++++ .../IDraftSnapshotKeeperFactory.cs | 12 ++++++ .../IFinalSnapshotCalculator.cs | 3 +- .../ISnapshotService.cs | 39 +++++++++++++++++++ .../Infrastructure/SnapshotService.cs | 5 ++- .../Modules/ServicesModule.cs | 4 ++ .../Services/FinalSnapshotCalculator.cs | 27 +++++++------ .../Workflow/EodCommandsHandler.cs | 8 ++-- .../Controllers/ServiceController.cs | 1 + 10 files changed, 107 insertions(+), 44 deletions(-) create mode 100644 src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs create mode 100644 src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs create mode 100644 src/MarginTrading.Backend.Services/ISnapshotService.cs diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs index c73667bb0..40dcaf670 100644 --- a/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs +++ b/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs @@ -9,30 +9,5 @@ namespace MarginTrading.Backend.Core.Services { - public interface ISnapshotService - { - /// - /// Make final trading snapshot from current system state - /// - /// - /// - /// - /// - Task MakeTradingDataSnapshot( - DateTime tradingDay, - string correlationId, - SnapshotStatus status = SnapshotStatus.Final); - - /// - /// Make final trading snapshot from draft - /// - /// - /// - /// - /// - Task MakeTradingDataSnapshotFromDraft( - string correlationId, - IEnumerable cfdQuotes, - IEnumerable fxRates); - } + } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs b/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs new file mode 100644 index 000000000..50a574efa --- /dev/null +++ b/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs @@ -0,0 +1,25 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using MarginTrading.Backend.Core.Repositories; + +namespace MarginTrading.Backend.Services +{ + public class DraftSnapshotKeeperFactory : IDraftSnapshotKeeperFactory + { + private readonly ITradingEngineSnapshotsRepository _tradingEngineSnapshotsRepository; + + public DraftSnapshotKeeperFactory(ITradingEngineSnapshotsRepository tradingEngineSnapshotsRepository) + { + _tradingEngineSnapshotsRepository = tradingEngineSnapshotsRepository; + } + + public IDraftSnapshotKeeper Create(DateTime tradingDay) + { + var draftSnapshotKeeper = new DraftSnapshotKeeper(_tradingEngineSnapshotsRepository); + draftSnapshotKeeper.Init(tradingDay); + return draftSnapshotKeeper; + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs b/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs new file mode 100644 index 000000000..d31b77592 --- /dev/null +++ b/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs @@ -0,0 +1,12 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Services +{ + public interface IDraftSnapshotKeeperFactory + { + IDraftSnapshotKeeper Create(DateTime tradingDay); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs b/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs index 99e5a448f..34ad1cc56 100644 --- a/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs +++ b/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs @@ -23,6 +23,7 @@ public interface IFinalSnapshotCalculator Task RunAsync( IEnumerable fxRates, IEnumerable cfdQuotes, - string correlationId); + string correlationId, + IDraftSnapshotKeeper draftSnapshotKeeper = null); } } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/ISnapshotService.cs b/src/MarginTrading.Backend.Services/ISnapshotService.cs new file mode 100644 index 000000000..83d02ca3e --- /dev/null +++ b/src/MarginTrading.Backend.Services/ISnapshotService.cs @@ -0,0 +1,39 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using MarginTrading.Backend.Contracts.Prices; +using MarginTrading.Backend.Core.Snapshots; + +namespace MarginTrading.Backend.Services +{ + public interface ISnapshotService + { + /// + /// Make final trading snapshot from current system state + /// + /// + /// + /// + /// + Task MakeTradingDataSnapshot( + DateTime tradingDay, + string correlationId, + SnapshotStatus status = SnapshotStatus.Final); + + /// + /// Make final trading snapshot from draft + /// + /// + /// + /// + /// + Task MakeTradingDataSnapshotFromDraft( + string correlationId, + IEnumerable cfdQuotes, + IEnumerable fxRates, + IDraftSnapshotKeeper draftSnapshotKeeper = null); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs index 0e0100030..103462178 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs @@ -219,7 +219,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho public async Task MakeTradingDataSnapshotFromDraft( string correlationId, IEnumerable cfdQuotes, - IEnumerable fxRates) + IEnumerable fxRates, + IDraftSnapshotKeeper draftSnapshotKeeper = null) { if (IsMakingSnapshotInProgress) { @@ -229,7 +230,7 @@ public async Task MakeTradingDataSnapshotFromDraft( await Lock.WaitAsync(); try { - var snapshot = await _finalSnapshotCalculator.RunAsync(fxRates, cfdQuotes, correlationId); + var snapshot = await _finalSnapshotCalculator.RunAsync(fxRates, cfdQuotes, correlationId, draftSnapshotKeeper); await _tradingEngineSnapshotsRepository.AddAsync(snapshot); } finally diff --git a/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs b/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs index dae8e36fa..c2a54311d 100644 --- a/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs +++ b/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs @@ -223,6 +223,10 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .InstancePerLifetimeScope(); + + builder.RegisterType() + .As() + .SingleInstance(); builder.RegisterType().As().SingleInstance(); diff --git a/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs b/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs index bca15b7d5..932866103 100644 --- a/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs +++ b/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs @@ -42,8 +42,13 @@ public FinalSnapshotCalculator(ICfdCalculatorService cfdCalculatorService, } /// - public async Task RunAsync(IEnumerable fxRates, IEnumerable cfdQuotes, string correlationId) + public async Task RunAsync(IEnumerable fxRates, + IEnumerable cfdQuotes, + string correlationId, + IDraftSnapshotKeeper draftSnapshotKeeper = null) { + var keeper = draftSnapshotKeeper ?? _draftSnapshotKeeper; + var fxRatesList = fxRates?.ToList(); var cfdQuotesList = cfdQuotes?.ToList(); @@ -53,14 +58,14 @@ public async Task RunAsync(IEnumerable fxR if (cfdQuotesList == null || !cfdQuotesList.Any()) throw new EmptyPriceUploadException(); - var positions = _draftSnapshotKeeper.GetPositions(); - var accounts = (await _draftSnapshotKeeper.GetAccountsAsync()).ToImmutableArray(); + var positions = keeper.GetPositions(); + var accounts = (await keeper.GetAccountsAsync()).ToImmutableArray(); foreach (var closingFxRate in fxRatesList) { ApplyFxRate(positions, accounts, closingFxRate.ClosePrice, closingFxRate.AssetId); } - var orders = _draftSnapshotKeeper.GetAllOrders(); + var orders = keeper.GetAllOrders(); foreach (var closingAssetPrice in cfdQuotesList) { ApplyCfdQuote(positions, orders, accounts, closingAssetPrice.ClosePrice, closingAssetPrice.AssetId); @@ -68,7 +73,7 @@ public async Task RunAsync(IEnumerable fxR var quotesTimestamp = _dateService.Now(); - await _draftSnapshotKeeper.UpdateAsync( + await keeper.UpdateAsync( positions, orders, accounts, @@ -76,14 +81,14 @@ await _draftSnapshotKeeper.UpdateAsync( cfdQuotesList.Select(q => q.ToContract(quotesTimestamp)) ); - return new TradingEngineSnapshot(_draftSnapshotKeeper.TradingDay, + return new TradingEngineSnapshot(keeper.TradingDay, correlationId, - _draftSnapshotKeeper.Timestamp, - MapToFinalJson(orders, _draftSnapshotKeeper), - MapToFinalJson(positions, _draftSnapshotKeeper), + keeper.Timestamp, + MapToFinalJson(orders, keeper), + MapToFinalJson(positions, keeper), await MapToFinalJson(accounts), - MapToJson(_draftSnapshotKeeper.FxPrices), - MapToJson(_draftSnapshotKeeper.CfdQuotes), + MapToJson(keeper.FxPrices), + MapToJson(keeper.CfdQuotes), SnapshotStatus.Final); } diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index c3c255cfa..c794f35d3 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -25,20 +25,20 @@ public class EodCommandsHandler private readonly IQuotesApi _quotesApi; private readonly ISnapshotService _snapshotService; private readonly IDateService _dateService; - private readonly IDraftSnapshotKeeper _draftSnapshotKeeper; + private readonly IDraftSnapshotKeeperFactory _draftSnapshotKeeperFactory; private readonly ILog _log; public EodCommandsHandler( IQuotesApi quotesApi, ISnapshotService snapshotService, IDateService dateService, - IDraftSnapshotKeeper draftSnapshotKeeper, + IDraftSnapshotKeeperFactory draftSnapshotKeeperFactory, ILog log) { _quotesApi = quotesApi; _snapshotService = snapshotService; _dateService = dateService; - _draftSnapshotKeeper = draftSnapshotKeeper; + _draftSnapshotKeeperFactory = draftSnapshotKeeperFactory; _log = log; } @@ -55,7 +55,7 @@ private async Task Handle(CreateSnapshotCommand command, IEventPublisher publish throw new Exception($"Could not receive quotes from BookKeeper: {quotes.ErrorCode.ToString()}"); } - _draftSnapshotKeeper.Init(command.TradingDay); + var draftSnapshotKeeper = _draftSnapshotKeeperFactory.Create(command.TradingDay); await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, MapQuotes(quotes.EodMarketData.Underlyings), diff --git a/src/MarginTrading.Backend/Controllers/ServiceController.cs b/src/MarginTrading.Backend/Controllers/ServiceController.cs index 737da7f24..786e1ef42 100644 --- a/src/MarginTrading.Backend/Controllers/ServiceController.cs +++ b/src/MarginTrading.Backend/Controllers/ServiceController.cs @@ -10,6 +10,7 @@ using MarginTrading.Backend.Core.Repositories; using MarginTrading.Backend.Core.Services; using MarginTrading.Backend.Extensions; +using MarginTrading.Backend.Services; using MarginTrading.Backend.Services.TradingConditions; using MarginTrading.Common.Middleware; using Microsoft.AspNetCore.Authorization; From 798b336822d74167c0c4a915fe02799bc2bd32bc Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Tue, 19 Mar 2024 14:09:23 +0500 Subject: [PATCH 13/14] fix(LT-5012): pass draftSnapshotKeeper --- .../Workflow/EodCommandsHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index c794f35d3..ccf280f97 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -59,7 +59,8 @@ private async Task Handle(CreateSnapshotCommand command, IEventPublisher publish await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, MapQuotes(quotes.EodMarketData.Underlyings), - MapFxRates(quotes.EodMarketData.Forex)); + MapFxRates(quotes.EodMarketData.Forex), + draftSnapshotKeeper); publisher.PublishEvent(new SnapshotCreatedEvent { From 7fee16ac0b96b0d57099e455dcdf120bdefec6f4 Mon Sep 17 00:00:00 2001 From: Gennadii Ponomarev Date: Tue, 26 Mar 2024 12:10:14 +0500 Subject: [PATCH 14/14] fix(LT-5012): naming, pr review fixes --- .../Services/ISnapshotMonitor.cs | 15 -------- .../Services/ISnapshotStatusTracker.cs | 20 +++++++++++ .../Settings/SnapshotMonitorSettings.cs | 7 ++-- .../ScheduleSettingsCacheService.cs | 8 ++--- .../Infrastructure/SnapshotService.cs | 10 +++--- ...hotMonitor.cs => SnapshotStatusTracker.cs} | 34 +++++++++++-------- .../Modules/ManagersModule.cs | 4 +-- ...ervice.cs => SnapshotMonitoringService.cs} | 25 ++++++++------ src/MarginTrading.Backend/Startup.cs | 2 +- tests/MarginTradingTests/AssetDayOffTests.cs | 2 +- 10 files changed, 69 insertions(+), 58 deletions(-) delete mode 100644 src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs create mode 100644 src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs rename src/MarginTrading.Backend.Services/Infrastructure/{SnapshotMonitor.cs => SnapshotStatusTracker.cs} (53%) rename src/MarginTrading.Backend.Services/{SnapshotMonitorService.cs => SnapshotMonitoringService.cs} (74%) diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs deleted file mode 100644 index 25c566b3f..000000000 --- a/src/MarginTrading.Backend.Core/Services/ISnapshotMonitor.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2019 Lykke Corp. -// See the LICENSE file in the project root for more information. - -using System; - -namespace MarginTrading.Backend.Core.Services -{ - public interface ISnapshotMonitor - { - void SnapshotRequested(DateTime tradingDay); - void SnapshotInProgress(); - void SnapshotFinished(); - bool ShouldRetrySnapshot(out DateTime tradingDay); - } -} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs new file mode 100644 index 000000000..bf6283d45 --- /dev/null +++ b/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs @@ -0,0 +1,20 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Core.Services +{ + /// + /// This status tracker decouples draft snapshot creation from rabbit mq + /// Normal flow: generated => handled in PlatformClosureProjection => snapshot saved + /// Degraded flow: generated, snapshot requested => event not received in PlatformClosureProjection => SnapshotMonitoringService retries snapshot creation after a timeout + /// + public interface ISnapshotStatusTracker + { + void SnapshotRequested(DateTime tradingDay); + void SnapshotInProgress(); + void SnapshotCreated(); + bool ShouldRetrySnapshot(out DateTime tradingDay); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs index b0c961d07..e0c769647 100644 --- a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs +++ b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs @@ -8,14 +8,13 @@ namespace MarginTrading.Backend.Core.Settings public class SnapshotMonitorSettings { /// - /// Defines the interval between consecutive checks performed by the SnapshotMonitoring service. + /// Defines the interval between consecutive checks performed by the SnapshotMonitoringService service. /// public TimeSpan MonitoringDelay { get; set; } = TimeSpan.FromSeconds(30); /// - /// If snapshot is not created after a specified amount of time, SnapshotMonitorService will retry this operation + /// If snapshot is not created after a specified amount of time, creation will be retried /// - - public TimeSpan SnapshotCreationTimeout { get; set; } = TimeSpan.FromMinutes(5); + public TimeSpan DelayBeforeFallbackSnapshot { get; set; } = TimeSpan.FromMinutes(5); } } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs index 2554c4067..0654af760 100644 --- a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs +++ b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs @@ -51,7 +51,7 @@ public class ScheduleSettingsCacheService : IScheduleSettingsCacheService private readonly ReaderWriterLockSlim _readerWriterLockSlim = new ReaderWriterLockSlim(); private readonly IFeatureManager _featureManager; - private readonly ISnapshotMonitor _snapshotMonitor; + private readonly ISnapshotStatusTracker _snapshotStatusTracker; public ScheduleSettingsCacheService( ICqrsSender cqrsSender, @@ -61,7 +61,7 @@ public ScheduleSettingsCacheService( ILog log, OvernightMarginSettings overnightMarginSettings, IFeatureManager featureManager, - ISnapshotMonitor snapshotMonitor) + ISnapshotStatusTracker snapshotStatusTracker) { _cqrsSender = cqrsSender; _scheduleSettingsApi = scheduleSettingsApi; @@ -70,7 +70,7 @@ public ScheduleSettingsCacheService( _log = log; _overnightMarginSettings = overnightMarginSettings; _featureManager = featureManager; - _snapshotMonitor = snapshotMonitor; + _snapshotStatusTracker = snapshotStatusTracker; } public async Task UpdateAllSettingsAsync() @@ -201,7 +201,7 @@ private void HandleMarketStateChangesUnsafe(DateTime currentTime, string[] marke if (ev.IsPlatformClosureEvent()) { - _snapshotMonitor.SnapshotRequested(now.Date); + _snapshotStatusTracker.SnapshotRequested(now.Date); } _cqrsSender.PublishEvent(ev); diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs index 103462178..bc2631d68 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs @@ -36,7 +36,7 @@ public class SnapshotService : ISnapshotService private readonly IMarginTradingBlobRepository _blobRepository; private readonly ILog _log; private readonly IFinalSnapshotCalculator _finalSnapshotCalculator; - private readonly ISnapshotMonitor _snapshotMonitor; + private readonly ISnapshotStatusTracker _snapshotStatusTracker; private readonly MarginTradingSettings _settings; private static readonly SemaphoreSlim Lock = new SemaphoreSlim(1, 1); @@ -55,7 +55,7 @@ public SnapshotService( IMarginTradingBlobRepository blobRepository, ILog log, IFinalSnapshotCalculator finalSnapshotCalculator, - ISnapshotMonitor snapshotMonitor, + ISnapshotStatusTracker snapshotStatusTracker, MarginTradingSettings settings) { _scheduleSettingsCacheService = scheduleSettingsCacheService; @@ -70,7 +70,7 @@ public SnapshotService( _blobRepository = blobRepository; _log = log; _finalSnapshotCalculator = finalSnapshotCalculator; - _snapshotMonitor = snapshotMonitor; + _snapshotStatusTracker = snapshotStatusTracker; _settings = settings; } @@ -126,7 +126,7 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho try { - _snapshotMonitor.SnapshotInProgress(); + _snapshotStatusTracker.SnapshotInProgress(); var orders = _orderReader.GetAllOrders(); var ordersJson = orders.Select(o => o.ConvertToSnapshotContract(_orderReader, status)).ToJson(); @@ -203,7 +203,7 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho await _tradingEngineSnapshotsRepository.AddAsync(snapshot); - _snapshotMonitor.SnapshotFinished(); + _snapshotStatusTracker.SnapshotCreated(); await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), $"Trading data snapshot was written to the storage. {msg}"); diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs similarity index 53% rename from src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs rename to src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs index 9029aae97..3ea0e9326 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotMonitor.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs @@ -7,19 +7,15 @@ namespace MarginTrading.Backend.Services.Infrastructure { - /// - /// This class decouples draft snapshot creation from rabbit mq - /// Normal flow: MarketStateChangedEvent generated => handled in PlatformClosureProjection => snapshot saved - /// Degraded flow: MarketStateChangedEvent generated, snapshot requested => event not received in PlatformClosureProjection => SnapshotMonitorService retries snapshot creation after a timeout - /// - public class SnapshotMonitor : ISnapshotMonitor + /// + public class SnapshotStatusTracker : ISnapshotStatusTracker { private readonly SnapshotMonitorSettings _settings; - private DraftSnapshotState _state = DraftSnapshotState.None; + private DraftSnapshotStatus _status = DraftSnapshotStatus.None; private DateTime _timestamp; private DateTime _tradingDay; - public SnapshotMonitor(SnapshotMonitorSettings settings) + public SnapshotStatusTracker(SnapshotMonitorSettings settings) { _settings = settings; } @@ -27,34 +23,42 @@ public SnapshotMonitor(SnapshotMonitorSettings settings) // TODO: concurrency / thread safety? public void SnapshotRequested(DateTime tradingDay) { - _state = DraftSnapshotState.Requested; + _status = DraftSnapshotStatus.Requested; _timestamp = DateTime.UtcNow; _tradingDay = tradingDay; } public void SnapshotInProgress() { - _state = DraftSnapshotState.InProgress; + if (_status != DraftSnapshotStatus.Requested) + { + return; + } + _status = DraftSnapshotStatus.InProgress; } - public void SnapshotFinished() + public void SnapshotCreated() { - _state = DraftSnapshotState.None; + if (_status != DraftSnapshotStatus.InProgress) + { + return; + } + _status = DraftSnapshotStatus.None; _timestamp = default; _tradingDay = default; } public bool ShouldRetrySnapshot(out DateTime tradingDay) { - var shouldRetry = _state == DraftSnapshotState.Requested && - _timestamp.Add(_settings.SnapshotCreationTimeout) <= DateTime.UtcNow; + var shouldRetry = _status == DraftSnapshotStatus.Requested && + _timestamp.Add(_settings.DelayBeforeFallbackSnapshot) <= DateTime.UtcNow; tradingDay = shouldRetry ? _tradingDay : default; return shouldRetry; } - private enum DraftSnapshotState + private enum DraftSnapshotStatus { None, Requested, diff --git a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs index 9a3f59983..a51b99c10 100644 --- a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs +++ b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs @@ -58,8 +58,8 @@ protected override void Load(ContainerBuilder builder) .As() .SingleInstance(); - builder.RegisterType() - .As() + builder.RegisterType() + .As() .SingleInstance(); } } diff --git a/src/MarginTrading.Backend.Services/SnapshotMonitorService.cs b/src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs similarity index 74% rename from src/MarginTrading.Backend.Services/SnapshotMonitorService.cs rename to src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs index c0c707c27..d419033b6 100644 --- a/src/MarginTrading.Backend.Services/SnapshotMonitorService.cs +++ b/src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs @@ -13,22 +13,25 @@ namespace MarginTrading.Backend.Services { - public class SnapshotMonitorService : BackgroundService + /// + /// Attempts to build a draft snapshot when the platform is degraded (specifically, when there's an issue with rabbitmq) + /// + public class SnapshotMonitoringService : BackgroundService { - private readonly ISnapshotMonitor _snapshotMonitor; + private readonly ISnapshotStatusTracker _snapshotStatusTracker; private readonly ISnapshotService _snapshotService; private readonly IIdentityGenerator _identityGenerator; private readonly SnapshotMonitorSettings _settings; - private readonly ILogger _logger; + private readonly ILogger _logger; - public SnapshotMonitorService( - ISnapshotMonitor snapshotMonitor, + public SnapshotMonitoringService( + ISnapshotStatusTracker snapshotStatusTracker, ISnapshotService snapshotService, IIdentityGenerator identityGenerator, SnapshotMonitorSettings settings, - ILogger logger) + ILogger logger) { - _snapshotMonitor = snapshotMonitor; + _snapshotStatusTracker = snapshotStatusTracker; _snapshotService = snapshotService; _identityGenerator = identityGenerator; _settings = settings; @@ -37,14 +40,14 @@ public SnapshotMonitorService( protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - _logger.LogInformation("{ServiceName} started", nameof(SnapshotMonitorService)); + _logger.LogInformation("{ServiceName} started", nameof(SnapshotMonitoringService)); while (!stoppingToken.IsCancellationRequested) { - if (_snapshotMonitor.ShouldRetrySnapshot(out var tradingDay)) + if (_snapshotStatusTracker.ShouldRetrySnapshot(out var tradingDay)) { _logger.LogWarning("{ServiceName}: Trading Snapshot Draft was requested, but timeout exceeded. Attempting to create the snapshot.", - nameof(SnapshotMonitorService)); + nameof(SnapshotMonitoringService)); try { @@ -53,7 +56,7 @@ await _snapshotService.MakeTradingDataSnapshot(tradingDay, SnapshotStatus.Draft); _logger.LogInformation("{ServiceName}: Trading Snapshot Draft was created", - nameof(SnapshotMonitorService)); + nameof(SnapshotMonitoringService)); } catch (Exception ex) { diff --git a/src/MarginTrading.Backend/Startup.cs b/src/MarginTrading.Backend/Startup.cs index 6a42a588f..23691cd0a 100644 --- a/src/MarginTrading.Backend/Startup.cs +++ b/src/MarginTrading.Backend/Startup.cs @@ -130,7 +130,7 @@ public void ConfigureServices(IServiceCollection services) SetupLoggers(Configuration, services, _mtSettingsManager, correlationContextAccessor); - services.AddHostedService(); + services.AddHostedService(); } [UsedImplicitly] diff --git a/tests/MarginTradingTests/AssetDayOffTests.cs b/tests/MarginTradingTests/AssetDayOffTests.cs index 5e8273281..8f0d8e397 100644 --- a/tests/MarginTradingTests/AssetDayOffTests.cs +++ b/tests/MarginTradingTests/AssetDayOffTests.cs @@ -329,7 +329,7 @@ private IAssetPairDayOffService ArrangeDayOffService(DateTime dateTime, new EmptyLog(), new OvernightMarginSettings(), Mock.Of(), - new SnapshotMonitor(new SnapshotMonitorSettings())); + new SnapshotStatusTracker(new SnapshotMonitorSettings())); scheduleSettingsCacheService.UpdateAllSettingsAsync().GetAwaiter().GetResult(); return new AssetPairDayOffService(scheduleSettingsCacheService);