Added an endpoint to get the logs based on range of block ids (#181)

* Added an endpoint to get the logs for list of blocks * Pagination Fix (#171) * Pagination Fix * Update CirrusMongoData.cs * Update CirrusMongoData.cs * Added a customer serializer for IDictionary<string,object> in case the object is a complex object * Fix for BSON to JSON conversion issue with long numbers * Removed unused services in controller * Added the missing block hash from the endpoint response * Added to block index count for the new index added * Added split of number of block index indexes count in the db in Cirrus and main code Co-authored-by: TheDude <[email protected]> Co-authored-by: YakupIpek <[email protected]>
block-core · Dec 7, 2022 · 59b9560 · 59b9560
1 parent 4835f90
commit 59b9560
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 42 deletions.
diff --git a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
 using Blockcore.Indexer.Cirrus.Client;
@@ -22,6 +23,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using MongoDB.Bson.Serialization;
 
 namespace Blockcore.Indexer.Cirrus
 {
@@ -87,6 +89,9 @@ public void ConfigureServices(IServiceCollection services)
          services.AddScoped<TaskRunner,SmartContractSyncRunner>();
 
          services.AddTransient<IBlockRewindOperation, CirrusBlockRewindOperation>();
+
+
+         BsonSerializer.RegisterSerializer(typeof(IDictionary<string, object>), new ComplexTypeSerializer());
       }
 
       private static IServiceCollection RegisterSmartContractBuilder(IServiceCollection collection)

diff --git a/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs b/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs
@@ -1,9 +1,6 @@
 using System.ComponentModel.DataAnnotations;
-using System.Linq;
 using System.Threading.Tasks;
 using Blockcore.Indexer.Cirrus.Storage;
-using Blockcore.Indexer.Cirrus.Storage.Mongo.SmartContracts;
-using Blockcore.Indexer.Cirrus.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Operations;
 using Blockcore.Indexer.Core.Paging;
 using Blockcore.Indexer.Core.Storage.Types;
@@ -20,18 +17,11 @@ public class CirrusQueryController : Controller
    {
       private readonly IPagingHelper paging;
       private readonly ICirrusStorage cirrusMongoData;
-      readonly IComputeSmartContractService<DaoContractTable> daoContractService;
-      readonly IComputeSmartContractService<StandardTokenContractTable> standardTokenService;
-      readonly IComputeSmartContractService<NonFungibleTokenContractTable> nonFungibleTokenService;
 
-      public CirrusQueryController(IPagingHelper paging,
-         IComputeSmartContractService<DaoContractTable> daoContractAggregator, ICirrusStorage cirrusMongoData, IComputeSmartContractService<StandardTokenContractTable> standardTokenService, IComputeSmartContractService<NonFungibleTokenContractTable> nonFungibleTokenService)
+      public CirrusQueryController(IPagingHelper paging, ICirrusStorage cirrusMongoData)
       {
          this.paging = paging;
-         daoContractService = daoContractAggregator;
          this.cirrusMongoData = cirrusMongoData;
-         this.standardTokenService = standardTokenService;
-         this.nonFungibleTokenService = nonFungibleTokenService;
       }
 
       [HttpGet]
@@ -48,6 +38,16 @@ public IActionResult GetContracts([MinLength(2)][MaxLength(100)] string contract
          return OkPaging(cirrusMongoData.ListContracts(contractType, offset, limit));
       }
 
+      [HttpGet]
+      [Route("contracts/logs")]
+      public IActionResult GetContracts([Range(0, long.MaxValue)] long startBlock,[Range(0, long.MaxValue)] long endBlock, [Range(0, long.MaxValue)] int? offset = 0, [Range(1, 1000)] int limit = 1000)
+      {
+         if (endBlock < startBlock)
+            return BadRequest();
+
+         return OkPaging(cirrusMongoData.ListBLocksLogs(startBlock,endBlock, offset, limit));
+      }
+
       [HttpGet]
       [Route("collectables/{ownerAddress}")]
       public IActionResult GetAddressAssets([MinLength(30)][MaxLength(100)] string ownerAddress, [Range(0, long.MaxValue)] int? offset = 0, [Range(1, 50)] int limit = 10)

diff --git a/src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs b/src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs
@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+
+namespace Blockcore.Indexer.Cirrus.Models;
+
+public class QueryBlockSmartContractsLogs
+{
+   public string TransactionHash { get; set; }
+   public string BlockHash { get; set; }
+   public long BlockNumber { get; set; }
+   public string PostState { get; set; }
+   public ulong GasUsed { get; set; }
+   public string From { get; set; }
+   public string To { get; set; }
+   public string NewContractAddress { get; set; }
+   public bool Success { get; set; }
+   public string ReturnValue { get; set; }
+   public string Bloom { get; set; }
+   public string Error { get; set; }
+
+   // public string MethodName { get; set; }
+   // public string ContractCodeType { get; set; }
+   // public string ContractBytecode { get; set; }
+   // public string ContractCodeHash { get; set; }
+   // public string ContractCSharp { get; set; }
+   // public ulong GasPrice { get; set; }
+   // public ulong Amount { get; set; }
+   // public ulong ContractBalance { get; set; }
+
+   public LogResponse[] Logs { get; set; }
+
+   public class LogResponse
+   {
+      public string Address { get; set; }
+      public string[] Topics { get; set; }
+      public string Data { get; set; }
+
+      public LogData Log { get; set; }
+   }
+
+   public class LogData
+   {
+      public string Event { get; set; }
+
+      public IDictionary<string, object> Data { get; set; }
+   }
+}
diff --git a/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs b/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs
@@ -14,6 +14,7 @@ public interface ICirrusStorage
    QueryContractCode ContractCode(string address);
    QueryResult<QueryContractGroup> GroupedContracts();
    QueryResult<QueryContractList> ListContracts(string contractType, int? offset, int limit);
+   QueryResult<QueryBlockSmartContractsLogs> ListBLocksLogs(long startBlock, long endBlock, int? offset, int limit);
 
    Task<QueryDAOContract> GetDaoContractByAddressAsync(string contractAddress);
    Task<QueryStandardTokenContract> GetStandardTokenContractByAddressAsync(string contractAddress);

diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs
@@ -2,16 +2,21 @@
 using Blockcore.Indexer.Cirrus.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage.Mongo;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using MongoDB.Driver;
 
 namespace Blockcore.Indexer.Cirrus.Storage.Mongo
 {
    public class CirrusMongoBuilder : MongoBuilder
    {
-      public CirrusMongoBuilder(ILogger<MongoBuilder> logger, IMongoDb data, IOptions<IndexerSettings> nakoConfiguration, IOptions<ChainSettings> chainSettings)
+      ICirrusMongoDb cirrusMongoDb;
+
+      public CirrusMongoBuilder(ILogger<MongoBuilder> logger, ICirrusMongoDb data, IOptions<IndexerSettings> nakoConfiguration, IOptions<ChainSettings> chainSettings)
          : base(logger, data, nakoConfiguration,chainSettings)
       {
+         cirrusMongoDb = data;
       }
 
       public override Task OnExecute()
@@ -44,6 +49,10 @@ public override Task OnExecute()
          SetDocumentMapAndIgnoreExtraElements<StandardTokenHolderTable>();
          SetDocumentMapAndIgnoreExtraElements<DaoContractProposalTable>();
 
+         cirrusMongoDb.CirrusContractTable.Indexes
+            .CreateOne(new CreateIndexModel<CirrusContractTable>(Builders<CirrusContractTable>
+               .IndexKeys.Ascending(_ => _.BlockIndex))); //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain
+
          return Task.CompletedTask;
       }
 

diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using AutoMapper.Internal;
 using Blockcore.Indexer.Cirrus.Models;
 using Blockcore.Indexer.Cirrus.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Client;
@@ -56,10 +55,12 @@ public CirrusMongoData(
       protected override async Task OnDeleteBlockAsync(SyncBlockInfo block)
       {
          // delete the contracts
-         FilterDefinition<CirrusContractTable> contractFilter = Builders<CirrusContractTable>.Filter.Eq(info => info.BlockIndex, block.BlockIndex);
+         FilterDefinition<CirrusContractTable> contractFilter =
+            Builders<CirrusContractTable>.Filter.Eq(info => info.BlockIndex, block.BlockIndex);
          Task<DeleteResult> contracts = mongoDb.CirrusContractTable.DeleteManyAsync(contractFilter);
 
-         FilterDefinition<CirrusContractCodeTable> contractCodeFilter = Builders<CirrusContractCodeTable>.Filter.Eq(info => info.BlockIndex, block.BlockIndex);
+         FilterDefinition<CirrusContractCodeTable> contractCodeFilter =
+            Builders<CirrusContractCodeTable>.Filter.Eq(info => info.BlockIndex, block.BlockIndex);
          Task<DeleteResult> contractsCode = mongoDb.CirrusContractCodeTable.DeleteManyAsync(contractCodeFilter);
 
          await Task.WhenAll(contracts, contractsCode);
@@ -68,20 +69,16 @@ protected override async Task OnDeleteBlockAsync(SyncBlockInfo block)
       public QueryResult<QueryContractGroup> GroupedContracts()
       {
          var groupedContracts = mongoDb.CirrusContractCodeTable.Aggregate()
-            .Group(_ => _.CodeType, ac => new QueryContractGroup
-            {
-               ContractCodeType = ac.Key,
-               Count = ac.Count(),
-               ContractHash = ac.First().ContractHash
-            })
+            .Group(_ => _.CodeType,
+               ac => new QueryContractGroup
+               {
+                  ContractCodeType = ac.Key, Count = ac.Count(), ContractHash = ac.First().ContractHash
+               })
             .ToList();
 
          return new QueryResult<QueryContractGroup>
          {
-            Items = groupedContracts,
-            Offset = 0,
-            Limit = groupedContracts.Count,
-            Total = groupedContracts.Count
+            Items = groupedContracts, Offset = 0, Limit = groupedContracts.Count, Total = groupedContracts.Count
          };
       }
 
@@ -95,7 +92,7 @@ public QueryResult<QueryContractList> ListContracts(string contractType, int? of
          int itemsToSkip = offset ?? (total < limit ? 0 : total - limit);
 
          IMongoQueryable<CirrusContractTable> cirrusContract = mongoDb.CirrusContractTable.AsQueryable()
-            .Where(q => q.ContractOpcode == "create" &&  q.ContractCodeType == contractType && q.Success == true)
+            .Where(q => q.ContractOpcode == "create" && q.ContractCodeType == contractType && q.Success == true)
             .OrderBy(b => b.BlockIndex)
             .Skip(itemsToSkip)
             .Take(limit);
@@ -113,14 +110,52 @@ public QueryResult<QueryContractList> ListContracts(string contractType, int? of
 
          return new QueryResult<QueryContractList>
          {
-            Items = transactions,
-            Offset = itemsToSkip,
-            Limit = limit,
-            Total = total
+            Items = transactions, Offset = itemsToSkip, Limit = limit, Total = total
+         };
+      }
+
+      public QueryResult<QueryBlockSmartContractsLogs> ListBLocksLogs(long startBlock, long endBlock, int? offset, int limit)
+      {
+         var total = mongoDb.CirrusContractTable
+            .AsQueryable()
+            .Count(_ => _.BlockIndex >= startBlock && _.BlockIndex <= endBlock);
+
+         var contracts = mongoDb.CirrusContractTable.AsQueryable()
+            .Where(_ => _.BlockIndex >= startBlock && _.BlockIndex <= endBlock)
+            .Skip((offset ?? 0) * limit)
+            .Take(limit)
+            .ToList();
+
+         var response = contracts.Select(receipt =>
+            new QueryBlockSmartContractsLogs
+            {
+               NewContractAddress = receipt.NewContractAddress,
+               From = receipt.FromAddress,
+               To = receipt.ToAddress,
+               BlockNumber = receipt.BlockIndex,
+               TransactionHash = receipt.TransactionId,
+               Success = receipt.Success,
+               Error = receipt.Error,
+               PostState = receipt.PostState,
+               GasUsed = receipt.GasUsed,
+               BlockHash = receipt.BlockHash,
+               Logs = receipt.Logs.Select(l => new Blockcore.Indexer.Cirrus.Models.QueryBlockSmartContractsLogs.LogResponse
+               {
+                  Address = l.Address,Data = l.Data,Log = new QueryBlockSmartContractsLogs.LogData
+                  {
+                     Data = l.Log.Data,Event = l.Log.Event
+                  },
+                  Topics = l.Topics
+               }).ToArray()
+            });
+
+         return new QueryResult<QueryBlockSmartContractsLogs>
+         {
+            Items = response, Limit = limit, Offset = offset ?? 0, Total = total
          };
       }
 
-      public async Task<QueryDAOContract> GetDaoContractByAddressAsync(string contractAddress)
+   public async Task<QueryDAOContract> GetDaoContractByAddressAsync(string contractAddress)
       {
          var contract = await mongoDb.DaoContractTable.Find(_ => _.ContractAddress == contractAddress)
             .SingleOrDefaultAsync();

diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs
@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.IO;
+using MongoDB.Bson.Serialization;
+using MongoDB.Bson.Serialization.Serializers;
+using JsonConvert = Newtonsoft.Json.JsonConvert;
+
+namespace Blockcore.Indexer.Cirrus.Storage.Mongo;
+
+public class ComplexTypeSerializer : SerializerBase<object>
+{
+   public override object Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
+   {
+      var serializer = BsonSerializer.LookupSerializer(typeof(BsonDocument));
+      var document = serializer.Deserialize(context, args);
+
+      var bsonDocument = document.ToBsonDocument();
+
+      var result = BsonExtensionMethods.ToJson(bsonDocument,new JsonWriterSettings { OutputMode = JsonOutputMode.Strict });
+      return JsonConvert.DeserializeObject<IDictionary<string, object>>(result);
+   }
+
+   public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object value)
+   {
+      var jsonDocument = JsonConvert.SerializeObject(value);
+      var bsonDocument = BsonSerializer.Deserialize<BsonDocument>(jsonDocument);
+
+      var serializer = BsonSerializer.LookupSerializer(typeof(BsonDocument));
+      serializer.Serialize(context, bsonDocument.AsBsonValue);
+   }
+}
diff --git a/src/Blockcore.Indexer.Cirrus/appsettings.json b/src/Blockcore.Indexer.Cirrus/appsettings.json
@@ -46,6 +46,7 @@
       "NumberOfPullerTasksForIBD" : 2,
       "MaxItemsInBlockingCollection" : 1000,
       "MaxItemsInHistoryQueue" : 10,
-      "AverageInterval": 10
+      "AverageInterval": 10,
+      "IndexCountForBlockIndexProperty" : 7
    }
 }
diff --git a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs
@@ -40,5 +40,7 @@ public class IndexerSettings
       public int NumberOfPullerTasksForIBD { get; set; }
       public int MaxItemsInBlockingCollection { get; set; }
       public int MaxItemsInHistoryQueue { get; set; }
+
+      public int IndexCountForBlockIndexProperty { get;set; }
    }
 }
diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs
@@ -19,8 +19,6 @@ namespace Blockcore.Indexer.Core.Sync.SyncTasks
 {
    public class BlockIndexer : TaskRunner
    {
-      public const int ExpectedNumberOfIndexes = 6;
-
       private readonly IndexerSettings config;
       private readonly ILogger<BlockIndexer> log;
       readonly IStorage data;
@@ -269,9 +267,9 @@ await db.Mempool.Indexes
             {
                List<string> allIndexes = data.GetBlockIndexIndexes();
 
-               if (allIndexes.Count != ExpectedNumberOfIndexes)
+               if (allIndexes.Count != config.IndexCountForBlockIndexProperty)
                {
-                  throw new ApplicationException($"Expected {ExpectedNumberOfIndexes} indexes but got {allIndexes.Count}");
+                  throw new ApplicationException($"Expected {config.IndexCountForBlockIndexProperty} indexes but got {allIndexes.Count}");
                }
 
                Runner.GlobalState.IndexMode = false;

diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs
@@ -1,14 +1,15 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using Blockcore.Indexer.Core.Client;
 using Blockcore.Indexer.Core.Client.Types;
 using Blockcore.Indexer.Core.Operations;
 using Blockcore.Indexer.Core.Operations.Types;
+using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage;
 using Blockcore.Indexer.Core.Storage.Mongo;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace Blockcore.Indexer.Core.Sync.SyncTasks
 {
@@ -24,8 +25,8 @@ public class BlockStartup : TaskStarter
       private readonly SyncConnection connection;
       private readonly IStorageOperations storageOperations;
       readonly ICryptoClientFactory clientFactory;
-      readonly IStorage data;
       private readonly MongoData mongoData;
+      readonly IOptions<IndexerSettings> indexerSettings;
 
       /// <summary>
       /// Initializes a new instance of the <see cref="BlockStartup"/> class.
@@ -36,13 +37,14 @@ public BlockStartup(
          SyncConnection syncConnection,
          IStorageOperations storageOperations,
          ICryptoClientFactory clientFactory,
-         IStorage data)
+         IStorage data,
+         IOptions<IndexerSettings> indexerSettings)
           : base(logger)
       {
          connection = syncConnection;
          this.storageOperations = storageOperations;
          this.clientFactory = clientFactory;
-         this.data = data;
+         this.indexerSettings = indexerSettings;
          this.syncOperations = syncOperations;
          log = logger;
 
@@ -67,7 +69,7 @@ public override async Task OnExecute()
 
          List<string> allIndexes = mongoData.GetBlockIndexIndexes();
 
-         if (allIndexes.Count == BlockIndexer.ExpectedNumberOfIndexes)
+         if (allIndexes.Count == indexerSettings.Value.IndexCountForBlockIndexProperty)
          {
             Runner.GlobalState.IndexModeCompleted = true;
          }