diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs index c73667bb0..40dcaf670 100644 --- a/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs +++ b/src/MarginTrading.Backend.Core/Services/ISnapshotService.cs @@ -9,30 +9,5 @@ namespace MarginTrading.Backend.Core.Services { - public interface ISnapshotService - { - /// - /// Make final trading snapshot from current system state - /// - /// - /// - /// - /// - Task MakeTradingDataSnapshot( - DateTime tradingDay, - string correlationId, - SnapshotStatus status = SnapshotStatus.Final); - - /// - /// Make final trading snapshot from draft - /// - /// - /// - /// - /// - Task MakeTradingDataSnapshotFromDraft( - string correlationId, - IEnumerable cfdQuotes, - IEnumerable fxRates); - } + } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs b/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs new file mode 100644 index 000000000..bf6283d45 --- /dev/null +++ b/src/MarginTrading.Backend.Core/Services/ISnapshotStatusTracker.cs @@ -0,0 +1,20 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Core.Services +{ + /// + /// This status tracker decouples draft snapshot creation from rabbit mq + /// Normal flow: generated => handled in PlatformClosureProjection => snapshot saved + /// Degraded flow: generated, snapshot requested => event not received in PlatformClosureProjection => SnapshotMonitoringService retries snapshot creation after a timeout + /// + public interface ISnapshotStatusTracker + { + void SnapshotRequested(DateTime tradingDay); + void SnapshotInProgress(); + void SnapshotCreated(); + bool ShouldRetrySnapshot(out DateTime tradingDay); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs new file mode 100644 index 000000000..7874146c4 --- /dev/null +++ b/src/MarginTrading.Backend.Core/Settings/BookKeeperServiceClient.cs @@ -0,0 +1,18 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using JetBrains.Annotations; +using Lykke.SettingsReader.Attributes; + +namespace MarginTrading.Backend.Core.Settings +{ + [UsedImplicitly] + public class BookKeeperServiceClient + { + // isAlive check leads to a deadlock + public string ServiceUrl { get; set; } + + [Optional] + public string ApiKey { get; set; } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs index a6c054ba3..1187e34c4 100644 --- a/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs +++ b/src/MarginTrading.Backend.Core/Settings/MarginTradingSettings.cs @@ -5,6 +5,7 @@ using JetBrains.Annotations; using Lykke.Common.Chaos; using Lykke.SettingsReader.Attributes; +using MarginTrading.Backend.Core.Services; using MarginTrading.Common.RabbitMq; using MarginTrading.Common.Settings; @@ -129,5 +130,7 @@ public class MarginTradingSettings // todo: probably should be moved turned in to a feature flag [Optional] public bool PerformanceTrackerEnabled { get; set; } = false; + + [Optional] public SnapshotMonitorSettings SnapshotMonitorSettings { get; set; } = new SnapshotMonitorSettings(); } } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs new file mode 100644 index 000000000..e0c769647 --- /dev/null +++ b/src/MarginTrading.Backend.Core/Settings/SnapshotMonitorSettings.cs @@ -0,0 +1,20 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Core.Settings +{ + public class SnapshotMonitorSettings + { + /// + /// Defines the interval between consecutive checks performed by the SnapshotMonitoringService service. + /// + public TimeSpan MonitoringDelay { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// If snapshot is not created after a specified amount of time, creation will be retried + /// + public TimeSpan DelayBeforeFallbackSnapshot { get; set; } = TimeSpan.FromMinutes(5); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs index fea501868..0654af760 100644 --- a/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs +++ b/src/MarginTrading.Backend.Services/AssetPairs/ScheduleSettingsCacheService.cs @@ -19,6 +19,8 @@ using MarginTrading.Common.Services; using MarginTrading.AssetService.Contracts; using MarginTrading.AssetService.Contracts.Scheduling; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Services.Extensions; using Microsoft.FeatureManagement; using MoreLinq; @@ -49,6 +51,7 @@ public class ScheduleSettingsCacheService : IScheduleSettingsCacheService private readonly ReaderWriterLockSlim _readerWriterLockSlim = new ReaderWriterLockSlim(); private readonly IFeatureManager _featureManager; + private readonly ISnapshotStatusTracker _snapshotStatusTracker; public ScheduleSettingsCacheService( ICqrsSender cqrsSender, @@ -57,7 +60,8 @@ public ScheduleSettingsCacheService( IDateService dateService, ILog log, OvernightMarginSettings overnightMarginSettings, - IFeatureManager featureManager) + IFeatureManager featureManager, + ISnapshotStatusTracker snapshotStatusTracker) { _cqrsSender = cqrsSender; _scheduleSettingsApi = scheduleSettingsApi; @@ -66,6 +70,7 @@ public ScheduleSettingsCacheService( _log = log; _overnightMarginSettings = overnightMarginSettings; _featureManager = featureManager; + _snapshotStatusTracker = snapshotStatusTracker; } public async Task UpdateAllSettingsAsync() @@ -185,12 +190,20 @@ private void HandleMarketStateChangesUnsafe(DateTime currentTime, string[] marke .Where(x => marketIds.IsNullOrEmpty() || marketIds.Contains(x.Key))) { var newState = scheduleSettings.GetMarketState(marketId, currentTime); - _cqrsSender.PublishEvent(new MarketStateChangedEvent + + var now = _dateService.Now(); + var ev = new MarketStateChangedEvent { Id = marketId, IsEnabled = newState.IsEnabled, - EventTimestamp = _dateService.Now(), - }); + EventTimestamp = now, + }; + + if (ev.IsPlatformClosureEvent()) + { + _snapshotStatusTracker.SnapshotRequested(now.Date); + } + _cqrsSender.PublishEvent(ev); _marketStates[marketId] = newState; } diff --git a/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs b/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs new file mode 100644 index 000000000..50a574efa --- /dev/null +++ b/src/MarginTrading.Backend.Services/DraftSnapshotKeeperFactory.cs @@ -0,0 +1,25 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using MarginTrading.Backend.Core.Repositories; + +namespace MarginTrading.Backend.Services +{ + public class DraftSnapshotKeeperFactory : IDraftSnapshotKeeperFactory + { + private readonly ITradingEngineSnapshotsRepository _tradingEngineSnapshotsRepository; + + public DraftSnapshotKeeperFactory(ITradingEngineSnapshotsRepository tradingEngineSnapshotsRepository) + { + _tradingEngineSnapshotsRepository = tradingEngineSnapshotsRepository; + } + + public IDraftSnapshotKeeper Create(DateTime tradingDay) + { + var draftSnapshotKeeper = new DraftSnapshotKeeper(_tradingEngineSnapshotsRepository); + draftSnapshotKeeper.Init(tradingDay); + return draftSnapshotKeeper; + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs b/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs new file mode 100644 index 000000000..d31b77592 --- /dev/null +++ b/src/MarginTrading.Backend.Services/IDraftSnapshotKeeperFactory.cs @@ -0,0 +1,12 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; + +namespace MarginTrading.Backend.Services +{ + public interface IDraftSnapshotKeeperFactory + { + IDraftSnapshotKeeper Create(DateTime tradingDay); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs b/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs index 99e5a448f..34ad1cc56 100644 --- a/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs +++ b/src/MarginTrading.Backend.Services/IFinalSnapshotCalculator.cs @@ -23,6 +23,7 @@ public interface IFinalSnapshotCalculator Task RunAsync( IEnumerable fxRates, IEnumerable cfdQuotes, - string correlationId); + string correlationId, + IDraftSnapshotKeeper draftSnapshotKeeper = null); } } \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/ISnapshotService.cs b/src/MarginTrading.Backend.Services/ISnapshotService.cs new file mode 100644 index 000000000..83d02ca3e --- /dev/null +++ b/src/MarginTrading.Backend.Services/ISnapshotService.cs @@ -0,0 +1,39 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using MarginTrading.Backend.Contracts.Prices; +using MarginTrading.Backend.Core.Snapshots; + +namespace MarginTrading.Backend.Services +{ + public interface ISnapshotService + { + /// + /// Make final trading snapshot from current system state + /// + /// + /// + /// + /// + Task MakeTradingDataSnapshot( + DateTime tradingDay, + string correlationId, + SnapshotStatus status = SnapshotStatus.Final); + + /// + /// Make final trading snapshot from draft + /// + /// + /// + /// + /// + Task MakeTradingDataSnapshotFromDraft( + string correlationId, + IEnumerable cfdQuotes, + IEnumerable fxRates, + IDraftSnapshotKeeper draftSnapshotKeeper = null); + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs index 4605456a3..bc2631d68 100644 --- a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotService.cs @@ -36,6 +36,7 @@ public class SnapshotService : ISnapshotService private readonly IMarginTradingBlobRepository _blobRepository; private readonly ILog _log; private readonly IFinalSnapshotCalculator _finalSnapshotCalculator; + private readonly ISnapshotStatusTracker _snapshotStatusTracker; private readonly MarginTradingSettings _settings; private static readonly SemaphoreSlim Lock = new SemaphoreSlim(1, 1); @@ -54,6 +55,7 @@ public SnapshotService( IMarginTradingBlobRepository blobRepository, ILog log, IFinalSnapshotCalculator finalSnapshotCalculator, + ISnapshotStatusTracker snapshotStatusTracker, MarginTradingSettings settings) { _scheduleSettingsCacheService = scheduleSettingsCacheService; @@ -68,6 +70,7 @@ public SnapshotService( _blobRepository = blobRepository; _log = log; _finalSnapshotCalculator = finalSnapshotCalculator; + _snapshotStatusTracker = snapshotStatusTracker; _settings = settings; } @@ -123,6 +126,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho try { + _snapshotStatusTracker.SnapshotInProgress(); + var orders = _orderReader.GetAllOrders(); var ordersJson = orders.Select(o => o.ConvertToSnapshotContract(_orderReader, status)).ToJson(); await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), @@ -162,6 +167,11 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho var accountsJson = accountStats .Select(a => a.ConvertToSnapshotContract(accountsInLiquidation.Contains(a), status)) .ToJson(); + + // timestamp will be used as an eod border + // setting it as close as possible to accountStats retrieval + var timestamp = _dateService.Now(); + await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), $"Preparing data... {accountStats.Count} accounts prepared."); @@ -183,7 +193,7 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho var snapshot = new TradingEngineSnapshot( tradingDay, correlationId, - _dateService.Now(), + timestamp, ordersJson: ordersJson, positionsJson: positionsJson, accountsJson: accountsJson, @@ -192,6 +202,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho status: status); await _tradingEngineSnapshotsRepository.AddAsync(snapshot); + + _snapshotStatusTracker.SnapshotCreated(); await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapshot), $"Trading data snapshot was written to the storage. {msg}"); @@ -207,7 +219,8 @@ await _log.WriteInfoAsync(nameof(SnapshotService), nameof(MakeTradingDataSnapsho public async Task MakeTradingDataSnapshotFromDraft( string correlationId, IEnumerable cfdQuotes, - IEnumerable fxRates) + IEnumerable fxRates, + IDraftSnapshotKeeper draftSnapshotKeeper = null) { if (IsMakingSnapshotInProgress) { @@ -217,8 +230,7 @@ public async Task MakeTradingDataSnapshotFromDraft( await Lock.WaitAsync(); try { - var snapshot = await _finalSnapshotCalculator.RunAsync(fxRates, cfdQuotes, correlationId); - + var snapshot = await _finalSnapshotCalculator.RunAsync(fxRates, cfdQuotes, correlationId, draftSnapshotKeeper); await _tradingEngineSnapshotsRepository.AddAsync(snapshot); } finally diff --git a/src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs new file mode 100644 index 000000000..3ea0e9326 --- /dev/null +++ b/src/MarginTrading.Backend.Services/Infrastructure/SnapshotStatusTracker.cs @@ -0,0 +1,68 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Core.Settings; + +namespace MarginTrading.Backend.Services.Infrastructure +{ + /// + public class SnapshotStatusTracker : ISnapshotStatusTracker + { + private readonly SnapshotMonitorSettings _settings; + private DraftSnapshotStatus _status = DraftSnapshotStatus.None; + private DateTime _timestamp; + private DateTime _tradingDay; + + public SnapshotStatusTracker(SnapshotMonitorSettings settings) + { + _settings = settings; + } + + // TODO: concurrency / thread safety? + public void SnapshotRequested(DateTime tradingDay) + { + _status = DraftSnapshotStatus.Requested; + _timestamp = DateTime.UtcNow; + _tradingDay = tradingDay; + } + + public void SnapshotInProgress() + { + if (_status != DraftSnapshotStatus.Requested) + { + return; + } + _status = DraftSnapshotStatus.InProgress; + } + + public void SnapshotCreated() + { + if (_status != DraftSnapshotStatus.InProgress) + { + return; + } + _status = DraftSnapshotStatus.None; + _timestamp = default; + _tradingDay = default; + } + + public bool ShouldRetrySnapshot(out DateTime tradingDay) + { + var shouldRetry = _status == DraftSnapshotStatus.Requested && + _timestamp.Add(_settings.DelayBeforeFallbackSnapshot) <= DateTime.UtcNow; + + tradingDay = shouldRetry ? _tradingDay : default; + + return shouldRetry; + } + + private enum DraftSnapshotStatus + { + None, + Requested, + InProgress, + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj index 3a226390c..4a50e189e 100644 --- a/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj +++ b/src/MarginTrading.Backend.Services/MarginTrading.Backend.Services.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs index b71874307..a51b99c10 100644 --- a/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs +++ b/src/MarginTrading.Backend.Services/Modules/ManagersModule.cs @@ -57,6 +57,10 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .SingleInstance(); + + builder.RegisterType() + .As() + .SingleInstance(); } } } diff --git a/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs b/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs index dae8e36fa..c2a54311d 100644 --- a/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs +++ b/src/MarginTrading.Backend.Services/Modules/ServicesModule.cs @@ -223,6 +223,10 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .InstancePerLifetimeScope(); + + builder.RegisterType() + .As() + .SingleInstance(); builder.RegisterType().As().SingleInstance(); diff --git a/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs b/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs index bca15b7d5..932866103 100644 --- a/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs +++ b/src/MarginTrading.Backend.Services/Services/FinalSnapshotCalculator.cs @@ -42,8 +42,13 @@ public FinalSnapshotCalculator(ICfdCalculatorService cfdCalculatorService, } /// - public async Task RunAsync(IEnumerable fxRates, IEnumerable cfdQuotes, string correlationId) + public async Task RunAsync(IEnumerable fxRates, + IEnumerable cfdQuotes, + string correlationId, + IDraftSnapshotKeeper draftSnapshotKeeper = null) { + var keeper = draftSnapshotKeeper ?? _draftSnapshotKeeper; + var fxRatesList = fxRates?.ToList(); var cfdQuotesList = cfdQuotes?.ToList(); @@ -53,14 +58,14 @@ public async Task RunAsync(IEnumerable fxR if (cfdQuotesList == null || !cfdQuotesList.Any()) throw new EmptyPriceUploadException(); - var positions = _draftSnapshotKeeper.GetPositions(); - var accounts = (await _draftSnapshotKeeper.GetAccountsAsync()).ToImmutableArray(); + var positions = keeper.GetPositions(); + var accounts = (await keeper.GetAccountsAsync()).ToImmutableArray(); foreach (var closingFxRate in fxRatesList) { ApplyFxRate(positions, accounts, closingFxRate.ClosePrice, closingFxRate.AssetId); } - var orders = _draftSnapshotKeeper.GetAllOrders(); + var orders = keeper.GetAllOrders(); foreach (var closingAssetPrice in cfdQuotesList) { ApplyCfdQuote(positions, orders, accounts, closingAssetPrice.ClosePrice, closingAssetPrice.AssetId); @@ -68,7 +73,7 @@ public async Task RunAsync(IEnumerable fxR var quotesTimestamp = _dateService.Now(); - await _draftSnapshotKeeper.UpdateAsync( + await keeper.UpdateAsync( positions, orders, accounts, @@ -76,14 +81,14 @@ await _draftSnapshotKeeper.UpdateAsync( cfdQuotesList.Select(q => q.ToContract(quotesTimestamp)) ); - return new TradingEngineSnapshot(_draftSnapshotKeeper.TradingDay, + return new TradingEngineSnapshot(keeper.TradingDay, correlationId, - _draftSnapshotKeeper.Timestamp, - MapToFinalJson(orders, _draftSnapshotKeeper), - MapToFinalJson(positions, _draftSnapshotKeeper), + keeper.Timestamp, + MapToFinalJson(orders, keeper), + MapToFinalJson(positions, keeper), await MapToFinalJson(accounts), - MapToJson(_draftSnapshotKeeper.FxPrices), - MapToJson(_draftSnapshotKeeper.CfdQuotes), + MapToJson(keeper.FxPrices), + MapToJson(keeper.CfdQuotes), SnapshotStatus.Final); } diff --git a/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs b/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs index e152511da..95ab0d628 100644 --- a/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs +++ b/src/MarginTrading.Backend.Services/Settings/MtBackendSettings.cs @@ -31,6 +31,8 @@ public class MtBackendSettings public ExchangeConnectorServiceClient MtStpExchangeConnectorClient { get; set; } public SettingsServiceClient SettingsServiceClient { get; set; } + + public BookKeeperServiceClient BookKeeperServiceClient { get; set; } public AccountsManagementServiceClient AccountsManagementServiceClient { get; set; } diff --git a/src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs b/src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs new file mode 100644 index 000000000..d419033b6 --- /dev/null +++ b/src/MarginTrading.Backend.Services/SnapshotMonitoringService.cs @@ -0,0 +1,76 @@ +// Copyright (c) 2019 Lykke Corp. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using MarginTrading.Backend.Core.Repositories; +using MarginTrading.Backend.Core.Services; +using MarginTrading.Backend.Core.Settings; +using MarginTrading.Backend.Core.Snapshots; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace MarginTrading.Backend.Services +{ + /// + /// Attempts to build a draft snapshot when the platform is degraded (specifically, when there's an issue with rabbitmq) + /// + public class SnapshotMonitoringService : BackgroundService + { + private readonly ISnapshotStatusTracker _snapshotStatusTracker; + private readonly ISnapshotService _snapshotService; + private readonly IIdentityGenerator _identityGenerator; + private readonly SnapshotMonitorSettings _settings; + private readonly ILogger _logger; + + public SnapshotMonitoringService( + ISnapshotStatusTracker snapshotStatusTracker, + ISnapshotService snapshotService, + IIdentityGenerator identityGenerator, + SnapshotMonitorSettings settings, + ILogger logger) + { + _snapshotStatusTracker = snapshotStatusTracker; + _snapshotService = snapshotService; + _identityGenerator = identityGenerator; + _settings = settings; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("{ServiceName} started", nameof(SnapshotMonitoringService)); + + while (!stoppingToken.IsCancellationRequested) + { + if (_snapshotStatusTracker.ShouldRetrySnapshot(out var tradingDay)) + { + _logger.LogWarning("{ServiceName}: Trading Snapshot Draft was requested, but timeout exceeded. Attempting to create the snapshot.", + nameof(SnapshotMonitoringService)); + + try + { + await _snapshotService.MakeTradingDataSnapshot(tradingDay, + _identityGenerator.GenerateGuid(), + SnapshotStatus.Draft); + + _logger.LogInformation("{ServiceName}: Trading Snapshot Draft was created", + nameof(SnapshotMonitoringService)); + } + catch (Exception ex) + { + _logger.LogCritical(ex, + "Could not create trading data snapshot for {TradingDay}. {Message}", + tradingDay, + ex.Message); + + // exception is swallowed to allow retries + } + } + + await Task.Delay(_settings.MonitoringDelay, stoppingToken); + } + } + } +} \ No newline at end of file diff --git a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs index d137dd412..ccf280f97 100644 --- a/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs +++ b/src/MarginTrading.Backend.Services/Workflow/EodCommandsHandler.cs @@ -2,38 +2,65 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using BookKeeper.Client; +using BookKeeper.Client.Responses.Eod; using BookKeeper.Client.Workflow.Commands; using BookKeeper.Client.Workflow.Events; using Common.Log; using JetBrains.Annotations; using Lykke.Cqrs; +using MarginTrading.Backend.Contracts.Prices; using MarginTrading.Backend.Core.Services; using MarginTrading.Common.Services; +using Microsoft.Extensions.DependencyInjection; namespace MarginTrading.Backend.Services.Workflow { [UsedImplicitly] public class EodCommandsHandler { + private readonly IQuotesApi _quotesApi; private readonly ISnapshotService _snapshotService; private readonly IDateService _dateService; + private readonly IDraftSnapshotKeeperFactory _draftSnapshotKeeperFactory; private readonly ILog _log; - public EodCommandsHandler(ISnapshotService snapshotService, IDateService dateService, ILog log) + public EodCommandsHandler( + IQuotesApi quotesApi, + ISnapshotService snapshotService, + IDateService dateService, + IDraftSnapshotKeeperFactory draftSnapshotKeeperFactory, + ILog log) { + _quotesApi = quotesApi; _snapshotService = snapshotService; _dateService = dateService; + _draftSnapshotKeeperFactory = draftSnapshotKeeperFactory; _log = log; } [UsedImplicitly] private async Task Handle(CreateSnapshotCommand command, IEventPublisher publisher) { - //deduplication is inside _snapshotService.MakeTradingDataSnapshot + //deduplication is inside _snapshotService try { - await _snapshotService.MakeTradingDataSnapshot(command.TradingDay, command.OperationId); + var quotes = await _quotesApi.GetCfdQuotes(command.TradingDay); + + if (quotes.ErrorCode != EodMarketDataErrorCodesContract.None) + { + throw new Exception($"Could not receive quotes from BookKeeper: {quotes.ErrorCode.ToString()}"); + } + + var draftSnapshotKeeper = _draftSnapshotKeeperFactory.Create(command.TradingDay); + + await _snapshotService.MakeTradingDataSnapshotFromDraft(command.OperationId, + MapQuotes(quotes.EodMarketData.Underlyings), + MapFxRates(quotes.EodMarketData.Forex), + draftSnapshotKeeper); publisher.PublishEvent(new SnapshotCreatedEvent { @@ -54,5 +81,23 @@ await _log.WriteErrorAsync(nameof(EodCommandsHandler), nameof(CreateSnapshotComm }); } } + + private IEnumerable MapQuotes(IEnumerable bestPrices) + { + return bestPrices.Select(x => new ClosingAssetPrice() + { + ClosePrice = x.Ask, // equal to bid + AssetId = x.Id, + }); + } + + private IEnumerable MapFxRates(IEnumerable bestPrices) + { + return bestPrices.Select(x => new ClosingFxRate() + { + ClosePrice = x.Ask, // equal to bid + AssetId = x.Id, + }); + } } } \ No newline at end of file diff --git a/src/MarginTrading.Backend/Controllers/ServiceController.cs b/src/MarginTrading.Backend/Controllers/ServiceController.cs index 737da7f24..786e1ef42 100644 --- a/src/MarginTrading.Backend/Controllers/ServiceController.cs +++ b/src/MarginTrading.Backend/Controllers/ServiceController.cs @@ -10,6 +10,7 @@ using MarginTrading.Backend.Core.Repositories; using MarginTrading.Backend.Core.Services; using MarginTrading.Backend.Extensions; +using MarginTrading.Backend.Services; using MarginTrading.Backend.Services.TradingConditions; using MarginTrading.Common.Middleware; using Microsoft.AspNetCore.Authorization; diff --git a/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs b/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs index c86966973..f2481e0a9 100644 --- a/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs +++ b/src/MarginTrading.Backend/Modules/BackendSettingsModule.cs @@ -27,6 +27,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterInstance(_settings.CurrentValue.RiskInformingSettings ?? new RiskInformingSettings {Data = new RiskInformingParams[0]}).SingleInstance(); builder.RegisterInstance(_settings.CurrentValue.MtBackend.OvernightMargin).SingleInstance(); + builder.RegisterInstance(_settings.CurrentValue.MtBackend.SnapshotMonitorSettings).SingleInstance(); } } } diff --git a/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs b/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs index d76d5f2e4..33cd82429 100644 --- a/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs +++ b/src/MarginTrading.Backend/Modules/ExternalServicesModule.cs @@ -3,6 +3,7 @@ using System; using Autofac; +using BookKeeper.Client; using Common.Log; using Lykke.HttpClientGenerator; using Lykke.HttpClientGenerator.Retries; @@ -186,6 +187,16 @@ protected override void Load(ContainerBuilder builder) .As().SingleInstance(); #endregion OrderBook Service + + #region BookKeeper + + builder + .Register(ctx => BuildBookKeeperClientGenerator(ctx) + .Generate()) + .As() + .SingleInstance(); + + #endregion } private HttpClientGenerator BuildAccountManagementClientGenerator(IComponentContext ctx) @@ -222,5 +233,25 @@ private HttpClientGenerator BuildSettingsClientGenerator(IComponentContext ctx) return settingsClientGeneratorBuilder.Create(); } + + private HttpClientGenerator BuildBookKeeperClientGenerator(IComponentContext ctx) + { + var bookkeeperSettings = _settings.CurrentValue.BookKeeperServiceClient; + + var bookkeeperClientGeneratorBuilder = HttpClientGenerator + .BuildForUrl(bookkeeperSettings.ServiceUrl) + .WithAdditionalDelegatingHandler(ctx.Resolve()) + .WithServiceName( + $"BookKeeper [{bookkeeperSettings.ServiceUrl}]") + .WithRetriesStrategy(new LinearRetryStrategy(TimeSpan.FromMilliseconds(300), 3)); + + if (!string.IsNullOrWhiteSpace(bookkeeperSettings.ApiKey)) + { + bookkeeperClientGeneratorBuilder = bookkeeperClientGeneratorBuilder + .WithApiKey(bookkeeperSettings.ApiKey); + } + + return bookkeeperClientGeneratorBuilder.Create(); + } } } \ No newline at end of file diff --git a/src/MarginTrading.Backend/Startup.cs b/src/MarginTrading.Backend/Startup.cs index 0b3964447..23691cd0a 100644 --- a/src/MarginTrading.Backend/Startup.cs +++ b/src/MarginTrading.Backend/Startup.cs @@ -129,6 +129,8 @@ public void ConfigureServices(IServiceCollection services) services.AddFeatureManagement(_mtSettingsManager.CurrentValue.MtBackend); SetupLoggers(Configuration, services, _mtSettingsManager, correlationContextAccessor); + + services.AddHostedService(); } [UsedImplicitly] diff --git a/tests/MarginTradingTests/AssetDayOffTests.cs b/tests/MarginTradingTests/AssetDayOffTests.cs index 9b384cd12..8f0d8e397 100644 --- a/tests/MarginTradingTests/AssetDayOffTests.cs +++ b/tests/MarginTradingTests/AssetDayOffTests.cs @@ -328,7 +328,8 @@ private IAssetPairDayOffService ArrangeDayOffService(DateTime dateTime, dateService.Object, new EmptyLog(), new OvernightMarginSettings(), - Mock.Of()); + Mock.Of(), + new SnapshotStatusTracker(new SnapshotMonitorSettings())); scheduleSettingsCacheService.UpdateAllSettingsAsync().GetAwaiter().GetResult(); return new AssetPairDayOffService(scheduleSettingsCacheService); diff --git a/tests/MarginTradingTests/BaseTests.cs b/tests/MarginTradingTests/BaseTests.cs index f2ec1fa98..806db0fb1 100644 --- a/tests/MarginTradingTests/BaseTests.cs +++ b/tests/MarginTradingTests/BaseTests.cs @@ -80,10 +80,12 @@ private void RegisterDependenciesCore(bool mockEvents = false) }, ReportingEquivalentPricesSettings = new[] {new ReportingEquivalentPricesSettings {EquivalentAsset = "USD", LegalEntity = "LYKKETEST"}}, - OvernightMargin = overnightMarginSettings + OvernightMargin = overnightMarginSettings, + SnapshotMonitorSettings = new SnapshotMonitorSettings(), }; builder.RegisterInstance(marginSettings).SingleInstance(); + builder.RegisterInstance(marginSettings.SnapshotMonitorSettings).SingleInstance(); builder.RegisterInstance(PositionHistoryEvents).As>().SingleInstance(); builder.RegisterInstance(overnightMarginSettings).SingleInstance(); builder.RegisterInstance(Mock.Of());