Decoupling mongo db dependencies to allow adding postgreSQL (#213)

* Moved mongo service registration into mongo namespace

* Removed mongo dependency from storage operations class

* Moved the rich list scanning into mongo (this is actually only relevant for address based blockchains unlike BTC)

* The block indexing sync task is dedicated to mongo db and has been moved into the mongo namespace

* Moved Mongo driver dependency into mongo namespace

* Removed stats syncer and stats controller dependency on mongo namespace

* Removed mongo dependency from insight controller

* Removed mongo dependency from query controller

* Removed mongo dependency from QueryOrphanBlock.cs

* Removed mongo dependency from InsertStats class

* Commented out unused code that has a dependency on mongo namespace

* Renamed settings properties from prefix mongo to db

* Removed mongo dependency from core tasks

* Fixed text in controller

* Decoupled the storage batch from Mongo db dependency

* Deleted the utxo cache service that was not in use

README.md

@@ -66,8 +66,8 @@ The default configurations are read from [appsetting.json](src/Blockcore.Indexer
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "MongoBatchSize": 10000000,
-      "MongoBatchCount" : 10000,
+      "DbBatchSize": 10000000,
+      "DbBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,
@@ -226,4 +226,4 @@ Blockcore.Indexer.Client.BitcoinCommunicationException: Daemon Failed Url = 'htt
  ---> System.Net.Sockets.SocketException (111): Connection refused
 ```
 
-Issue: This happens when the DNS name is not accessible.
+Issue: This happens when the DNS name is not accessible.

src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs

@@ -1,4 +1,4 @@
-using Blockcore.Indexer.Core.Storage.Mongo.Types;
+using Blockcore.Indexer.Core.Storage.Types;
 
 namespace Blockcore.Indexer.Angor.Storage.Mongo.Types;
 

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs

@@ -3,6 +3,7 @@
 using Blockcore.Indexer.Angor.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
+using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.Indexer.Core.Sync.SyncTasks;
 using Blockcore.NBitcoin.DataEncoders;
 using Microsoft.Extensions.Options;

src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs

@@ -6,6 +6,7 @@
 using Blockcore.Indexer.Core.Operations.Types;
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
+using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.Indexer.Core.Sync.SyncTasks;
 using Blockcore.NBitcoin;
 using Blockcore.NBitcoin.BIP32;

src/Blockcore.Indexer.Angor/appsettings.json

@@ -18,6 +18,7 @@
       }
    },
    "Indexer": {
+      "DbType": "MongoDb",
       "ConnectionString": "mongodb://{Symbol}-mongo",
       "DatabaseNameSubfix": true,
 
@@ -34,8 +35,8 @@
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "MongoBatchSize": 10000000,
-      "MongoBatchCount" : 10000,
+      "DbBatchSize": 10000000,
+      "DbBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,

src/Blockcore.Indexer.Cirrus/CirrusStartup.cs

@@ -61,6 +61,7 @@ public void ConfigureServices(IServiceCollection services)
 
          services.AddSingleton<ICirrusStorage, CirrusMongoData>();
          services.AddSingleton<ICirrusMongoDb, CirrusMongoDb>();
+         services.Replace(new ServiceDescriptor(typeof(IStorageBatchFactory), typeof(CirrusStorageBatchFactory),ServiceLifetime.Singleton));
 
          services.AddControllers()
             .AddApplicationPart(typeof(Startup).Assembly)

src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs

@@ -4,11 +4,10 @@
 
 namespace Blockcore.Indexer.Cirrus.Operations.Types
 {
-   public class CirrusStorageBatch
+   public class CirrusStorageBatch : MongoStorageBatch
    {
-      public List<CirrusContractTable> CirrusContractTable { get; set; } = new List<CirrusContractTable>();
-
-      public List<CirrusContractCodeTable> CirrusContractCodeTable { get; set; } = new List<CirrusContractCodeTable>();
+      public List<CirrusContractTable> CirrusContractTable { get; set; } = new ();
 
+      public List<CirrusContractCodeTable> CirrusContractCodeTable { get; set; } = new ();
    }
 }

src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs

@@ -4,7 +4,7 @@
 using Blockcore.Indexer.Cirrus.Storage.Mongo.Types;
 using Blockcore.Indexer.Cirrus.Storage.Types;
 using Blockcore.Indexer.Core.Client.Types;
-using Blockcore.Indexer.Core.Storage;
+using Blockcore.Indexer.Core.Storage.Mongo;
 using Blockcore.Indexer.Core.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.NBitcoin;

src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs

@@ -28,7 +28,6 @@ public class CirrusMongoStorageOperations : MongoStorageOperations
       public CirrusMongoStorageOperations(
          SyncConnection syncConnection,
          IStorage storage,
-         IUtxoCache utxoCache,
          IOptions<IndexerSettings> configuration,
          GlobalState globalState,
          IMapMongoBlockToStorageBlock mongoBlockToStorageBlock,
@@ -38,7 +37,6 @@ public CirrusMongoStorageOperations(
          base(
              syncConnection,
              db,
-             utxoCache,
              configuration,
              globalState,
              mongoBlockToStorageBlock,
@@ -52,12 +50,7 @@ public CirrusMongoStorageOperations(
 
       protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item)
       {
-         storageBatch.ExtraData ??= new CirrusStorageBatch();
-
-         if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch))
-         {
-            throw new ArgumentNullException(nameof(cirrusStorageBatch));
-         }
+         CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch;
 
          foreach (Transaction transaction in item.Transactions)
          {
@@ -125,10 +118,7 @@ protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlock
 
       protected override void OnPushStorageBatch(StorageBatch storageBatch)
       {
-         if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch))
-         {
-            throw new ArgumentNullException(nameof(cirrusStorageBatch));
-         }
+         CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch;
 
          var t1 = Task.Run(() =>
          {

src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs

@@ -0,0 +1,10 @@
+using Blockcore.Indexer.Cirrus.Operations.Types;
+using Blockcore.Indexer.Core.Operations.Types;
+using Blockcore.Indexer.Core.Storage;
+
+namespace Blockcore.Indexer.Cirrus.Storage.Mongo;
+
+public class CirrusStorageBatchFactory : IStorageBatchFactory
+{
+   public StorageBatch GetStorageBatch() => new CirrusStorageBatch();
+}

src/Blockcore.Indexer.Cirrus/appsettings.json

@@ -18,7 +18,7 @@
       }
    },
    "Indexer": {
-      // "ConnectionString": "mongodb://localhost",
+      "DbType": "MongoDb",
       "ConnectionString": "mongodb://{Symbol}-mongo",
       "DatabaseNameSubfix": true,
 
@@ -34,8 +34,8 @@
       // Syncing parameters
       "SyncBlockchain": true,
       "SyncMemoryPool": true,
-      "MongoBatchSize": 10000000,
-      "MongoBatchCount" : 10000,
+      "DbBatchSize": 10000000,
+      "DbBatchCount" : 10000,
       "ParallelRequestsToTransactionRpc": 50,
       "DetailedTrace": 0,
       "MaxItemsInQueue": 10,

src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs

@@ -1,11 +1,9 @@
 using System;
 using System.Collections.Generic;
-using MongoDB.Bson.Serialization.Attributes;
 using Newtonsoft.Json;
 
 namespace Blockcore.Indexer.Core.Client.Types
 {
-   [BsonIgnoreExtraElements]
    public class PeerInfo
    {
       public string Addr { get; set; }

src/Blockcore.Indexer.Core/Controllers/InsightController.cs

@@ -172,7 +172,7 @@ private Supply CalculateSupply()
       /// <returns></returns>
       private List<Wallet> RetrieveWallets()
       {
-         // TODO: Funds should be stored in MongoDB, and be editable by individual chains and not hard-coded.
+         // TODO: Funds should be stored in the DB, and be editable by individual chains and not hard-coded.
          var funds = new List<Wallet>();
 
          List<Wallet> wallets = insightConfiguration.Wallets;
@@ -181,7 +181,7 @@ private List<Wallet> RetrieveWallets()
          {
             if (wallet.Address != null && wallet.Address.Length > 0)
             {
-               List<Storage.Mongo.Types.RichlistTable> balances = storage.AddressBalances(wallet.Address);
+               var balances = storage.AddressBalances(wallet.Address);
                long balance = balances.Sum(b => b.Balance);
                wallet.Balance = balance;
             }
@@ -207,7 +207,7 @@ private decimal CalculateCirculatingSupply()
             }
          }
 
-         List<Storage.Mongo.Types.RichlistTable> balances = storage.AddressBalances(listOfAddress);
+         var balances = storage.AddressBalances(listOfAddress);
          long walletBalances = balances.Sum(b => b.Balance);
 
          long circulatingSupply = totalBalance - walletBalances;

src/Blockcore.Indexer.Core/Controllers/QueryController.cs

@@ -79,7 +79,7 @@ public IActionResult GetAddressTransactions([MinLength(4)][MaxLength(100)] strin
       [Route("address/{address}/transactions/unspent")]
       public async Task<IActionResult> GetAddressTransactionsUnspent([MinLength(30)][MaxLength(100)] string address, long confirmations = 0, [Range(0, int.MaxValue)] int offset = 0, [Range(1, 50)] int limit = 10)
       {
-         QueryResult<Storage.Mongo.Types.OutputTable> result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit);
+         var result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit);
 
          return OkPaging(result);
       }

src/Blockcore.Indexer.Core/Controllers/StatsController.cs

@@ -1,10 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
+using Blockcore.Indexer.Core.Client.Types;
 using Blockcore.Indexer.Core.Handlers;
 using Blockcore.Indexer.Core.Models;
 using Blockcore.Indexer.Core.Storage;
-using Blockcore.Indexer.Core.Storage.Mongo;
 using Microsoft.AspNetCore.Mvc;
 
 namespace Blockcore.Indexer.Core.Controllers
@@ -18,15 +19,15 @@ public class StatsController : ControllerBase
    {
       private readonly StatsHandler statsHandler;
 
-      private readonly MongoData storage;
+      private readonly IStorage storage;
 
       /// <summary>
       /// Initializes a new instance of the <see cref="StatsController"/> class.
       /// </summary>
       public StatsController(StatsHandler statsHandler, IStorage storage)
       {
          this.statsHandler = statsHandler;
-         this.storage = storage as MongoData;
+         this.storage = storage;
       }
 
       [HttpGet]
@@ -83,7 +84,9 @@ public async Task<IActionResult> Peers()
       [Route("peers/{date}")]
       public IActionResult Peers(DateTime date)
       {
-         List<Client.Types.PeerInfo> list = storage.GetPeerFromDate(date);
+         List<PeerInfo> list = storage.GetPeerFromDate(date)
+            .Select(x => x as PeerInfo)
+            .ToList();
          return Ok(list);
       }
 

src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs

@@ -0,0 +1,7 @@
+namespace Blockcore.Indexer.Core.Models;
+
+public class BalanceForAddress
+{
+   public string Address { get; set; }
+   public long Balance { get; set; }
+}

src/Blockcore.Indexer.Core/Models/QueryBlock.cs

@@ -1,16 +1,9 @@
-using System.Collections.Generic;
 
 namespace Blockcore.Indexer.Core.Models
 {
-   public class QueryBlockResults
-   {
-      public IEnumerable<QueryBlock> Blocks { get; set; }
-
-      public int Total { get; set; }
-   }
-
    public class QueryBlock
    {
+
       /// <summary>
       /// Gets or sets the Symbol.
       /// </summary>
@@ -60,16 +53,16 @@ public class QueryBlock
 
       public string Bits { get; set; }
 
+      public double Difficulty { get; set; }
+
+      public string ChainWork { get; set; }
+
       public string Merkleroot { get; set; }
 
       public long Nonce { get; set; }
 
       public long Version { get; set; }
 
-      public double Difficulty { get; set; }
-
-      public string ChainWork { get; set; }
-
       public string PosBlockSignature { get; set; }
 
       public string PosModifierv2 { get; set; }
@@ -81,10 +74,5 @@ public class QueryBlock
       public string PosBlockTrust { get; set; }
 
       public string PosChainTrust { get; set; }
-
-      /// <summary>
-      /// Gets or sets the transactions.
-      /// </summary>
-      public IEnumerable<QueryTransaction> Transactions { get; set; }
    }
 }

src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+
+namespace Blockcore.Indexer.Core.Models;
+
+public class QueryBlockResults
+{
+   public IEnumerable<QueryBlock> Blocks { get; set; }
+
+   public int Total { get; set; }
+}

src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs

@@ -1,6 +1,5 @@
 using System;
-using System.Collections.Generic;
-using Blockcore.Indexer.Core.Storage.Mongo.Types;
+using Blockcore.Indexer.Core.Storage.Types;
 
 namespace Blockcore.Indexer.Core.Models
 {
@@ -9,6 +8,6 @@ public class QueryOrphanBlock
       public DateTime Created { get; set; }
       public uint BlockIndex { get; set; }
       public string BlockHash { get; set; }
-      public BlockTable Block { get; set; }
+      public QueryBlock Block { get; set; }
    }
 }

src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs

@@ -13,6 +13,6 @@ public interface IStorageOperations
 
       SyncBlockInfo PushStorageBatch(StorageBatch storageBatch);
 
-      InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item);
+      void InsertMempoolTransactions(SyncBlockTransactionsOperation item);
    }
 }

src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs

@@ -1,6 +1,3 @@
-using System.Collections.Generic;
-using Blockcore.Indexer.Core.Storage.Mongo.Types;
-
 namespace Blockcore.Indexer.Core.Operations.Types
 {
    #region Using Directives
@@ -23,10 +20,5 @@ public class InsertStats
       /// Gets or sets the outputs.
       /// </summary>
       public int InputsOutputs { get; set; }
-
-      /// <summary>
-      /// Gets or sets the items.
-      /// </summary>
-      public List<MempoolTable> Items { get; set; }
    }
 }