From 59b9560c814b1e7a7b2beddf40c568675fd0395f Mon Sep 17 00:00:00 2001 From: TheDude Date: Wed, 7 Dec 2022 14:24:52 +0000 Subject: [PATCH] Added an endpoint to get the logs based on range of block ids (#181) * Added an endpoint to get the logs for list of blocks * Pagination Fix (#171) * Pagination Fix * Update CirrusMongoData.cs * Update CirrusMongoData.cs * Added a customer serializer for IDictionary in case the object is a complex object * Fix for BSON to JSON conversion issue with long numbers * Removed unused services in controller * Added the missing block hash from the endpoint response * Added to block index count for the new index added * Added split of number of block index indexes count in the db in Cirrus and main code Co-authored-by: TheDude Co-authored-by: YakupIpek --- src/Blockcore.Indexer.Cirrus/CirrusStartup.cs | 5 ++ .../Controllers/CirrusQueryController.cs | 22 +++--- .../Models/QueryBlockSmartContractsLogs.cs | 46 ++++++++++++ .../Storage/ICirrusStorage.cs | 1 + .../Storage/Mongo/CirrusMongoBuilder.cs | 11 ++- .../Storage/Mongo/CirrusMongoData.cs | 73 ++++++++++++++----- .../Storage/Mongo/ComplexTypeSerializer.cs | 31 ++++++++ src/Blockcore.Indexer.Cirrus/appsettings.json | 3 +- .../Settings/IndexerSettings.cs | 2 + .../Sync/SyncTasks/BlockIndexer.cs | 6 +- .../Sync/SyncTasks/BlockStartup.cs | 12 +-- src/Blockcore.Indexer/appsettings.json | 3 +- 12 files changed, 173 insertions(+), 42 deletions(-) create mode 100644 src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs create mode 100644 src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs diff --git a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs index 67fed2ac..46de4c78 100644 --- a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs +++ b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Reflection; using Blockcore.Indexer.Cirrus.Client; @@ -22,6 +23,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using MongoDB.Bson.Serialization; namespace Blockcore.Indexer.Cirrus { @@ -87,6 +89,9 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddTransient(); + + + BsonSerializer.RegisterSerializer(typeof(IDictionary), new ComplexTypeSerializer()); } private static IServiceCollection RegisterSmartContractBuilder(IServiceCollection collection) diff --git a/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs b/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs index e6389733..912b8333 100644 --- a/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs +++ b/src/Blockcore.Indexer.Cirrus/Controllers/CirrusQueryController.cs @@ -1,9 +1,6 @@ using System.ComponentModel.DataAnnotations; -using System.Linq; using System.Threading.Tasks; using Blockcore.Indexer.Cirrus.Storage; -using Blockcore.Indexer.Cirrus.Storage.Mongo.SmartContracts; -using Blockcore.Indexer.Cirrus.Storage.Mongo.Types; using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Paging; using Blockcore.Indexer.Core.Storage.Types; @@ -20,18 +17,11 @@ public class CirrusQueryController : Controller { private readonly IPagingHelper paging; private readonly ICirrusStorage cirrusMongoData; - readonly IComputeSmartContractService daoContractService; - readonly IComputeSmartContractService standardTokenService; - readonly IComputeSmartContractService nonFungibleTokenService; - public CirrusQueryController(IPagingHelper paging, - IComputeSmartContractService daoContractAggregator, ICirrusStorage cirrusMongoData, IComputeSmartContractService standardTokenService, IComputeSmartContractService nonFungibleTokenService) + public CirrusQueryController(IPagingHelper paging, ICirrusStorage cirrusMongoData) { this.paging = paging; - daoContractService = daoContractAggregator; this.cirrusMongoData = cirrusMongoData; - this.standardTokenService = standardTokenService; - this.nonFungibleTokenService = nonFungibleTokenService; } [HttpGet] @@ -48,6 +38,16 @@ public IActionResult GetContracts([MinLength(2)][MaxLength(100)] string contract return OkPaging(cirrusMongoData.ListContracts(contractType, offset, limit)); } + [HttpGet] + [Route("contracts/logs")] + public IActionResult GetContracts([Range(0, long.MaxValue)] long startBlock,[Range(0, long.MaxValue)] long endBlock, [Range(0, long.MaxValue)] int? offset = 0, [Range(1, 1000)] int limit = 1000) + { + if (endBlock < startBlock) + return BadRequest(); + + return OkPaging(cirrusMongoData.ListBLocksLogs(startBlock,endBlock, offset, limit)); + } + [HttpGet] [Route("collectables/{ownerAddress}")] public IActionResult GetAddressAssets([MinLength(30)][MaxLength(100)] string ownerAddress, [Range(0, long.MaxValue)] int? offset = 0, [Range(1, 50)] int limit = 10) diff --git a/src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs b/src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs new file mode 100644 index 00000000..90d540bb --- /dev/null +++ b/src/Blockcore.Indexer.Cirrus/Models/QueryBlockSmartContractsLogs.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; + +namespace Blockcore.Indexer.Cirrus.Models; + +public class QueryBlockSmartContractsLogs +{ + public string TransactionHash { get; set; } + public string BlockHash { get; set; } + public long BlockNumber { get; set; } + public string PostState { get; set; } + public ulong GasUsed { get; set; } + public string From { get; set; } + public string To { get; set; } + public string NewContractAddress { get; set; } + public bool Success { get; set; } + public string ReturnValue { get; set; } + public string Bloom { get; set; } + public string Error { get; set; } + + // public string MethodName { get; set; } + // public string ContractCodeType { get; set; } + // public string ContractBytecode { get; set; } + // public string ContractCodeHash { get; set; } + // public string ContractCSharp { get; set; } + // public ulong GasPrice { get; set; } + // public ulong Amount { get; set; } + // public ulong ContractBalance { get; set; } + + public LogResponse[] Logs { get; set; } + + public class LogResponse + { + public string Address { get; set; } + public string[] Topics { get; set; } + public string Data { get; set; } + + public LogData Log { get; set; } + } + + public class LogData + { + public string Event { get; set; } + + public IDictionary Data { get; set; } + } +} diff --git a/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs b/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs index be95afa9..12be1de4 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/ICirrusStorage.cs @@ -14,6 +14,7 @@ public interface ICirrusStorage QueryContractCode ContractCode(string address); QueryResult GroupedContracts(); QueryResult ListContracts(string contractType, int? offset, int limit); + QueryResult ListBLocksLogs(long startBlock, long endBlock, int? offset, int limit); Task GetDaoContractByAddressAsync(string contractAddress); Task GetStandardTokenContractByAddressAsync(string contractAddress); diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs index 7d82afaa..8fd13efd 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoBuilder.cs @@ -2,16 +2,21 @@ using Blockcore.Indexer.Cirrus.Storage.Mongo.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Storage.Mongo.Types; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using MongoDB.Driver; namespace Blockcore.Indexer.Cirrus.Storage.Mongo { public class CirrusMongoBuilder : MongoBuilder { - public CirrusMongoBuilder(ILogger logger, IMongoDb data, IOptions nakoConfiguration, IOptions chainSettings) + ICirrusMongoDb cirrusMongoDb; + + public CirrusMongoBuilder(ILogger logger, ICirrusMongoDb data, IOptions nakoConfiguration, IOptions chainSettings) : base(logger, data, nakoConfiguration,chainSettings) { + cirrusMongoDb = data; } public override Task OnExecute() @@ -44,6 +49,10 @@ public override Task OnExecute() SetDocumentMapAndIgnoreExtraElements(); SetDocumentMapAndIgnoreExtraElements(); + cirrusMongoDb.CirrusContractTable.Indexes + .CreateOne(new CreateIndexModel(Builders + .IndexKeys.Ascending(_ => _.BlockIndex))); //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain + return Task.CompletedTask; } diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs index 8450fff8..287251c3 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoData.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using AutoMapper.Internal; using Blockcore.Indexer.Cirrus.Models; using Blockcore.Indexer.Cirrus.Storage.Mongo.Types; using Blockcore.Indexer.Core.Client; @@ -56,10 +55,12 @@ public CirrusMongoData( protected override async Task OnDeleteBlockAsync(SyncBlockInfo block) { // delete the contracts - FilterDefinition contractFilter = Builders.Filter.Eq(info => info.BlockIndex, block.BlockIndex); + FilterDefinition contractFilter = + Builders.Filter.Eq(info => info.BlockIndex, block.BlockIndex); Task contracts = mongoDb.CirrusContractTable.DeleteManyAsync(contractFilter); - FilterDefinition contractCodeFilter = Builders.Filter.Eq(info => info.BlockIndex, block.BlockIndex); + FilterDefinition contractCodeFilter = + Builders.Filter.Eq(info => info.BlockIndex, block.BlockIndex); Task contractsCode = mongoDb.CirrusContractCodeTable.DeleteManyAsync(contractCodeFilter); await Task.WhenAll(contracts, contractsCode); @@ -68,20 +69,16 @@ protected override async Task OnDeleteBlockAsync(SyncBlockInfo block) public QueryResult GroupedContracts() { var groupedContracts = mongoDb.CirrusContractCodeTable.Aggregate() - .Group(_ => _.CodeType, ac => new QueryContractGroup - { - ContractCodeType = ac.Key, - Count = ac.Count(), - ContractHash = ac.First().ContractHash - }) + .Group(_ => _.CodeType, + ac => new QueryContractGroup + { + ContractCodeType = ac.Key, Count = ac.Count(), ContractHash = ac.First().ContractHash + }) .ToList(); return new QueryResult { - Items = groupedContracts, - Offset = 0, - Limit = groupedContracts.Count, - Total = groupedContracts.Count + Items = groupedContracts, Offset = 0, Limit = groupedContracts.Count, Total = groupedContracts.Count }; } @@ -95,7 +92,7 @@ public QueryResult ListContracts(string contractType, int? of int itemsToSkip = offset ?? (total < limit ? 0 : total - limit); IMongoQueryable cirrusContract = mongoDb.CirrusContractTable.AsQueryable() - .Where(q => q.ContractOpcode == "create" && q.ContractCodeType == contractType && q.Success == true) + .Where(q => q.ContractOpcode == "create" && q.ContractCodeType == contractType && q.Success == true) .OrderBy(b => b.BlockIndex) .Skip(itemsToSkip) .Take(limit); @@ -113,14 +110,52 @@ public QueryResult ListContracts(string contractType, int? of return new QueryResult { - Items = transactions, - Offset = itemsToSkip, - Limit = limit, - Total = total + Items = transactions, Offset = itemsToSkip, Limit = limit, Total = total + }; + } + + public QueryResult ListBLocksLogs(long startBlock, long endBlock, int? offset, int limit) + { + var total = mongoDb.CirrusContractTable + .AsQueryable() + .Count(_ => _.BlockIndex >= startBlock && _.BlockIndex <= endBlock); + + var contracts = mongoDb.CirrusContractTable.AsQueryable() + .Where(_ => _.BlockIndex >= startBlock && _.BlockIndex <= endBlock) + .Skip((offset ?? 0) * limit) + .Take(limit) + .ToList(); + + var response = contracts.Select(receipt => + new QueryBlockSmartContractsLogs + { + NewContractAddress = receipt.NewContractAddress, + From = receipt.FromAddress, + To = receipt.ToAddress, + BlockNumber = receipt.BlockIndex, + TransactionHash = receipt.TransactionId, + Success = receipt.Success, + Error = receipt.Error, + PostState = receipt.PostState, + GasUsed = receipt.GasUsed, + BlockHash = receipt.BlockHash, + Logs = receipt.Logs.Select(l => new Blockcore.Indexer.Cirrus.Models.QueryBlockSmartContractsLogs.LogResponse + { + Address = l.Address,Data = l.Data,Log = new QueryBlockSmartContractsLogs.LogData + { + Data = l.Log.Data,Event = l.Log.Event + }, + Topics = l.Topics + }).ToArray() + }); + + return new QueryResult + { + Items = response, Limit = limit, Offset = offset ?? 0, Total = total }; } - public async Task GetDaoContractByAddressAsync(string contractAddress) + public async Task GetDaoContractByAddressAsync(string contractAddress) { var contract = await mongoDb.DaoContractTable.Find(_ => _.ContractAddress == contractAddress) .SingleOrDefaultAsync(); diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs new file mode 100644 index 00000000..a7163f1f --- /dev/null +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/ComplexTypeSerializer.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; +using JsonConvert = Newtonsoft.Json.JsonConvert; + +namespace Blockcore.Indexer.Cirrus.Storage.Mongo; + +public class ComplexTypeSerializer : SerializerBase +{ + public override object Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + var serializer = BsonSerializer.LookupSerializer(typeof(BsonDocument)); + var document = serializer.Deserialize(context, args); + + var bsonDocument = document.ToBsonDocument(); + + var result = BsonExtensionMethods.ToJson(bsonDocument,new JsonWriterSettings { OutputMode = JsonOutputMode.Strict }); + return JsonConvert.DeserializeObject>(result); + } + + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object value) + { + var jsonDocument = JsonConvert.SerializeObject(value); + var bsonDocument = BsonSerializer.Deserialize(jsonDocument); + + var serializer = BsonSerializer.LookupSerializer(typeof(BsonDocument)); + serializer.Serialize(context, bsonDocument.AsBsonValue); + } +} diff --git a/src/Blockcore.Indexer.Cirrus/appsettings.json b/src/Blockcore.Indexer.Cirrus/appsettings.json index f7df9afa..7e6e450b 100644 --- a/src/Blockcore.Indexer.Cirrus/appsettings.json +++ b/src/Blockcore.Indexer.Cirrus/appsettings.json @@ -46,6 +46,7 @@ "NumberOfPullerTasksForIBD" : 2, "MaxItemsInBlockingCollection" : 1000, "MaxItemsInHistoryQueue" : 10, - "AverageInterval": 10 + "AverageInterval": 10, + "IndexCountForBlockIndexProperty" : 7 } } diff --git a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs index 67a63527..5ccf3251 100644 --- a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs +++ b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs @@ -40,5 +40,7 @@ public class IndexerSettings public int NumberOfPullerTasksForIBD { get; set; } public int MaxItemsInBlockingCollection { get; set; } public int MaxItemsInHistoryQueue { get; set; } + + public int IndexCountForBlockIndexProperty { get;set; } } } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs index 59c30aac..80051f76 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs @@ -19,8 +19,6 @@ namespace Blockcore.Indexer.Core.Sync.SyncTasks { public class BlockIndexer : TaskRunner { - public const int ExpectedNumberOfIndexes = 6; - private readonly IndexerSettings config; private readonly ILogger log; readonly IStorage data; @@ -269,9 +267,9 @@ await db.Mempool.Indexes { List allIndexes = data.GetBlockIndexIndexes(); - if (allIndexes.Count != ExpectedNumberOfIndexes) + if (allIndexes.Count != config.IndexCountForBlockIndexProperty) { - throw new ApplicationException($"Expected {ExpectedNumberOfIndexes} indexes but got {allIndexes.Count}"); + throw new ApplicationException($"Expected {config.IndexCountForBlockIndexProperty} indexes but got {allIndexes.Count}"); } Runner.GlobalState.IndexMode = false; diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs index e48fea10..c49ebdf0 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs @@ -1,14 +1,15 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Blockcore.Indexer.Core.Client; using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Blockcore.Indexer.Core.Sync.SyncTasks { @@ -24,8 +25,8 @@ public class BlockStartup : TaskStarter private readonly SyncConnection connection; private readonly IStorageOperations storageOperations; readonly ICryptoClientFactory clientFactory; - readonly IStorage data; private readonly MongoData mongoData; + readonly IOptions indexerSettings; /// /// Initializes a new instance of the class. @@ -36,13 +37,14 @@ public BlockStartup( SyncConnection syncConnection, IStorageOperations storageOperations, ICryptoClientFactory clientFactory, - IStorage data) + IStorage data, + IOptions indexerSettings) : base(logger) { connection = syncConnection; this.storageOperations = storageOperations; this.clientFactory = clientFactory; - this.data = data; + this.indexerSettings = indexerSettings; this.syncOperations = syncOperations; log = logger; @@ -67,7 +69,7 @@ public override async Task OnExecute() List allIndexes = mongoData.GetBlockIndexIndexes(); - if (allIndexes.Count == BlockIndexer.ExpectedNumberOfIndexes) + if (allIndexes.Count == indexerSettings.Value.IndexCountForBlockIndexProperty) { Runner.GlobalState.IndexModeCompleted = true; } diff --git a/src/Blockcore.Indexer/appsettings.json b/src/Blockcore.Indexer/appsettings.json index 1b424e4a..c67bc07e 100644 --- a/src/Blockcore.Indexer/appsettings.json +++ b/src/Blockcore.Indexer/appsettings.json @@ -44,6 +44,7 @@ // Store the trx hex in mongo storage or read it from RPC "StoreRawTransactions": true, "NumberOfPullerTasksForIBD" : 5, - "MaxItemsInBlockingCollection" : 1000 + "MaxItemsInBlockingCollection" : 1000, + "IndexCountForBlockIndexProperty": 6 } }