From 6d09e34b5356fea40153505fd878d950076ba17f Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Thu, 11 May 2023 17:34:05 -0600 Subject: [PATCH] add gnosis support --- bchain/coins/blockchain.go | 3 + bchain/coins/eth/ethaddrcontracts.pb.go | 365 ++++++++++++++++++++ bchain/coins/eth/ethaddrcontracts.proto | 22 ++ bchain/coins/eth/ethrpc.go | 52 +-- bchain/coins/gnosis/evm.go | 141 ++++++++ bchain/coins/gnosis/gnosisrpc.go | 203 +++++++++++ bchain/coins/gnosis/types.go | 349 +++++++++++++++++++ blockbook.go | 9 +- configs/coins/gnosis.json | 72 ++++ configs/coins/gnosis_archive.json | 75 ++++ configs/coins/gnosis_archive_consensus.json | 38 ++ configs/coins/gnosis_consensus.json | 38 ++ db/bulkconnect.go | 42 +-- db/rocksdb.go | 68 +++- db/rocksdb_ethereumtype.go | 115 ++++-- docs/ports.md | 158 ++++----- 16 files changed, 1569 insertions(+), 181 deletions(-) create mode 100644 bchain/coins/eth/ethaddrcontracts.pb.go create mode 100644 bchain/coins/eth/ethaddrcontracts.proto create mode 100644 bchain/coins/gnosis/evm.go create mode 100644 bchain/coins/gnosis/gnosisrpc.go create mode 100644 bchain/coins/gnosis/types.go create mode 100644 configs/coins/gnosis.json create mode 100644 configs/coins/gnosis_archive.json create mode 100644 configs/coins/gnosis_archive_consensus.json create mode 100644 configs/coins/gnosis_consensus.json diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 31de78764a..8ffdd99df0 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -32,6 +32,7 @@ import ( "github.com/trezor/blockbook/bchain/coins/flo" "github.com/trezor/blockbook/bchain/coins/fujicoin" "github.com/trezor/blockbook/bchain/coins/gamecredits" + "github.com/trezor/blockbook/bchain/coins/gnosis" "github.com/trezor/blockbook/bchain/coins/grs" "github.com/trezor/blockbook/bchain/coins/koto" "github.com/trezor/blockbook/bchain/coins/liquid" @@ -138,6 +139,8 @@ func init() { BlockChainFactories["BNB Smart Chain Archive"] = bsc.NewBNBSmartChainRPC BlockChainFactories["Polygon"] = polygon.NewPolygonRPC BlockChainFactories["Polygon Archive"] = polygon.NewPolygonRPC + BlockChainFactories["Gnosis"] = gnosis.NewGnosisRPC + BlockChainFactories["Gnosis Archive"] = gnosis.NewGnosisRPC } // NewBlockChain creates bchain.BlockChain and bchain.Mempool for the coin passed by the parameter coin diff --git a/bchain/coins/eth/ethaddrcontracts.pb.go b/bchain/coins/eth/ethaddrcontracts.pb.go new file mode 100644 index 0000000000..a57f6687a6 --- /dev/null +++ b/bchain/coins/eth/ethaddrcontracts.pb.go @@ -0,0 +1,365 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v4.25.3 +// source: bchain/coins/eth/ethaddrcontracts.proto + +package eth + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ProtoAddrContracts struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TotalTxs uint64 `protobuf:"varint,1,opt,name=TotalTxs,proto3" json:"TotalTxs,omitempty"` + NonContractTxs uint64 `protobuf:"varint,2,opt,name=NonContractTxs,proto3" json:"NonContractTxs,omitempty"` + InternalTxs uint64 `protobuf:"varint,3,opt,name=InternalTxs,proto3" json:"InternalTxs,omitempty"` + Contracts []*ProtoAddrContracts_AddrContract `protobuf:"bytes,4,rep,name=Contracts,proto3" json:"Contracts,omitempty"` +} + +func (x *ProtoAddrContracts) Reset() { + *x = ProtoAddrContracts{} + if protoimpl.UnsafeEnabled { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProtoAddrContracts) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtoAddrContracts) ProtoMessage() {} + +func (x *ProtoAddrContracts) ProtoReflect() protoreflect.Message { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProtoAddrContracts.ProtoReflect.Descriptor instead. +func (*ProtoAddrContracts) Descriptor() ([]byte, []int) { + return file_bchain_coins_eth_ethaddrcontracts_proto_rawDescGZIP(), []int{0} +} + +func (x *ProtoAddrContracts) GetTotalTxs() uint64 { + if x != nil { + return x.TotalTxs + } + return 0 +} + +func (x *ProtoAddrContracts) GetNonContractTxs() uint64 { + if x != nil { + return x.NonContractTxs + } + return 0 +} + +func (x *ProtoAddrContracts) GetInternalTxs() uint64 { + if x != nil { + return x.InternalTxs + } + return 0 +} + +func (x *ProtoAddrContracts) GetContracts() []*ProtoAddrContracts_AddrContract { + if x != nil { + return x.Contracts + } + return nil +} + +type ProtoAddrContracts_MultiTokenValue struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id []byte `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"` +} + +func (x *ProtoAddrContracts_MultiTokenValue) Reset() { + *x = ProtoAddrContracts_MultiTokenValue{} + if protoimpl.UnsafeEnabled { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProtoAddrContracts_MultiTokenValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtoAddrContracts_MultiTokenValue) ProtoMessage() {} + +func (x *ProtoAddrContracts_MultiTokenValue) ProtoReflect() protoreflect.Message { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProtoAddrContracts_MultiTokenValue.ProtoReflect.Descriptor instead. +func (*ProtoAddrContracts_MultiTokenValue) Descriptor() ([]byte, []int) { + return file_bchain_coins_eth_ethaddrcontracts_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *ProtoAddrContracts_MultiTokenValue) GetId() []byte { + if x != nil { + return x.Id + } + return nil +} + +func (x *ProtoAddrContracts_MultiTokenValue) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +type ProtoAddrContracts_AddrContract struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type int64 `protobuf:"varint,1,opt,name=Type,proto3" json:"Type,omitempty"` + Contract []byte `protobuf:"bytes,2,opt,name=Contract,proto3" json:"Contract,omitempty"` + Txs uint64 `protobuf:"varint,3,opt,name=Txs,proto3" json:"Txs,omitempty"` + Value []byte `protobuf:"bytes,4,opt,name=Value,proto3" json:"Value,omitempty"` + Ids [][]byte `protobuf:"bytes,5,rep,name=Ids,proto3" json:"Ids,omitempty"` + MultiTokenValues []*ProtoAddrContracts_MultiTokenValue `protobuf:"bytes,6,rep,name=MultiTokenValues,proto3" json:"MultiTokenValues,omitempty"` +} + +func (x *ProtoAddrContracts_AddrContract) Reset() { + *x = ProtoAddrContracts_AddrContract{} + if protoimpl.UnsafeEnabled { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProtoAddrContracts_AddrContract) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtoAddrContracts_AddrContract) ProtoMessage() {} + +func (x *ProtoAddrContracts_AddrContract) ProtoReflect() protoreflect.Message { + mi := &file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProtoAddrContracts_AddrContract.ProtoReflect.Descriptor instead. +func (*ProtoAddrContracts_AddrContract) Descriptor() ([]byte, []int) { + return file_bchain_coins_eth_ethaddrcontracts_proto_rawDescGZIP(), []int{0, 1} +} + +func (x *ProtoAddrContracts_AddrContract) GetType() int64 { + if x != nil { + return x.Type + } + return 0 +} + +func (x *ProtoAddrContracts_AddrContract) GetContract() []byte { + if x != nil { + return x.Contract + } + return nil +} + +func (x *ProtoAddrContracts_AddrContract) GetTxs() uint64 { + if x != nil { + return x.Txs + } + return 0 +} + +func (x *ProtoAddrContracts_AddrContract) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *ProtoAddrContracts_AddrContract) GetIds() [][]byte { + if x != nil { + return x.Ids + } + return nil +} + +func (x *ProtoAddrContracts_AddrContract) GetMultiTokenValues() []*ProtoAddrContracts_MultiTokenValue { + if x != nil { + return x.MultiTokenValues + } + return nil +} + +var File_bchain_coins_eth_ethaddrcontracts_proto protoreflect.FileDescriptor + +var file_bchain_coins_eth_ethaddrcontracts_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x62, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x2f, 0x63, 0x6f, 0x69, 0x6e, 0x73, 0x2f, 0x65, + 0x74, 0x68, 0x2f, 0x65, 0x74, 0x68, 0x61, 0x64, 0x64, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbf, 0x03, 0x0a, 0x12, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x41, 0x64, 0x64, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, + 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x78, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x08, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x78, 0x73, 0x12, 0x26, 0x0a, 0x0e, + 0x4e, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x54, 0x78, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x4e, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, + 0x74, 0x54, 0x78, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x54, 0x78, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x54, 0x78, 0x73, 0x12, 0x3e, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, + 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x41, 0x64, 0x64, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2e, 0x41, + 0x64, 0x64, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x52, 0x09, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x1a, 0x37, 0x0a, 0x0f, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, + 0xc9, 0x01, 0x0a, 0x0c, 0x41, 0x64, 0x64, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x12, 0x10, 0x0a, 0x03, 0x54, 0x78, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x54, + 0x78, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x64, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x03, 0x49, 0x64, 0x73, 0x12, 0x4f, 0x0a, 0x10, 0x4d, 0x75, + 0x6c, 0x74, 0x69, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x41, 0x64, 0x64, 0x72, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x4d, 0x75, 0x6c, 0x74, 0x69, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x42, 0x2e, 0x5a, 0x2c, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x65, 0x7a, 0x6f, 0x72, + 0x2f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x62, 0x6f, 0x6f, 0x6b, 0x2f, 0x62, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x2f, 0x63, 0x6f, 0x69, 0x6e, 0x73, 0x2f, 0x65, 0x74, 0x68, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_bchain_coins_eth_ethaddrcontracts_proto_rawDescOnce sync.Once + file_bchain_coins_eth_ethaddrcontracts_proto_rawDescData = file_bchain_coins_eth_ethaddrcontracts_proto_rawDesc +) + +func file_bchain_coins_eth_ethaddrcontracts_proto_rawDescGZIP() []byte { + file_bchain_coins_eth_ethaddrcontracts_proto_rawDescOnce.Do(func() { + file_bchain_coins_eth_ethaddrcontracts_proto_rawDescData = protoimpl.X.CompressGZIP(file_bchain_coins_eth_ethaddrcontracts_proto_rawDescData) + }) + return file_bchain_coins_eth_ethaddrcontracts_proto_rawDescData +} + +var file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_bchain_coins_eth_ethaddrcontracts_proto_goTypes = []interface{}{ + (*ProtoAddrContracts)(nil), // 0: ProtoAddrContracts + (*ProtoAddrContracts_MultiTokenValue)(nil), // 1: ProtoAddrContracts.MultiTokenValue + (*ProtoAddrContracts_AddrContract)(nil), // 2: ProtoAddrContracts.AddrContract +} +var file_bchain_coins_eth_ethaddrcontracts_proto_depIdxs = []int32{ + 2, // 0: ProtoAddrContracts.Contracts:type_name -> ProtoAddrContracts.AddrContract + 1, // 1: ProtoAddrContracts.AddrContract.MultiTokenValues:type_name -> ProtoAddrContracts.MultiTokenValue + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_bchain_coins_eth_ethaddrcontracts_proto_init() } +func file_bchain_coins_eth_ethaddrcontracts_proto_init() { + if File_bchain_coins_eth_ethaddrcontracts_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProtoAddrContracts); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProtoAddrContracts_MultiTokenValue); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProtoAddrContracts_AddrContract); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_bchain_coins_eth_ethaddrcontracts_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_bchain_coins_eth_ethaddrcontracts_proto_goTypes, + DependencyIndexes: file_bchain_coins_eth_ethaddrcontracts_proto_depIdxs, + MessageInfos: file_bchain_coins_eth_ethaddrcontracts_proto_msgTypes, + }.Build() + File_bchain_coins_eth_ethaddrcontracts_proto = out.File + file_bchain_coins_eth_ethaddrcontracts_proto_rawDesc = nil + file_bchain_coins_eth_ethaddrcontracts_proto_goTypes = nil + file_bchain_coins_eth_ethaddrcontracts_proto_depIdxs = nil +} diff --git a/bchain/coins/eth/ethaddrcontracts.proto b/bchain/coins/eth/ethaddrcontracts.proto new file mode 100644 index 0000000000..4402ba6fc7 --- /dev/null +++ b/bchain/coins/eth/ethaddrcontracts.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +option go_package = "github.com/trezor/blockbook/bchain/coins/eth"; + +message ProtoAddrContracts { + message MultiTokenValue { + bytes Id = 1; + bytes Value = 2; + } + message AddrContract { + int64 Type = 1; + bytes Contract = 2; + uint64 Txs = 3; + bytes Value = 4; + repeated bytes Ids = 5; + repeated MultiTokenValue MultiTokenValues = 6; + } + uint64 TotalTxs = 1; + uint64 NonContractTxs = 2; + uint64 InternalTxs = 3; + repeated AddrContract Contracts = 4; +} \ No newline at end of file diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index a56b36666d..0cb094f285 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -54,26 +54,27 @@ type Configuration struct { // EthereumRPC is an interface to JSON-RPC eth service. type EthereumRPC struct { *bchain.BaseChain - Client bchain.EVMClient - RPC bchain.EVMRPCClient - MainNetChainID Network - Timeout time.Duration - Parser *EthereumParser - PushHandler func(bchain.NotificationType) - OpenRPC func(string) (bchain.EVMRPCClient, bchain.EVMClient, error) - Mempool *bchain.MempoolEthereumType - mempoolInitialized bool - bestHeaderLock sync.Mutex - bestHeader bchain.EVMHeader - bestHeaderTime time.Time - NewBlock bchain.EVMNewBlockSubscriber - newBlockSubscription bchain.EVMClientSubscription - NewTx bchain.EVMNewTxSubscriber - newTxSubscription bchain.EVMClientSubscription - ChainConfig *Configuration - supportedStakingPools []string - stakingPoolNames []string - stakingPoolContracts []string + Client bchain.EVMClient + RPC bchain.EVMRPCClient + MainNetChainID Network + Timeout time.Duration + Parser *EthereumParser + PushHandler func(bchain.NotificationType) + OpenRPC func(string) (bchain.EVMRPCClient, bchain.EVMClient, error) + GetInternalDataForBlock func(blockHash string, blockHeight uint32, transactions []bchain.RpcTransaction) ([]bchain.EthereumInternalData, []bchain.ContractInfo, error) + Mempool *bchain.MempoolEthereumType + mempoolInitialized bool + bestHeaderLock sync.Mutex + bestHeader bchain.EVMHeader + bestHeaderTime time.Time + NewBlock bchain.EVMNewBlockSubscriber + newBlockSubscription bchain.EVMClientSubscription + NewTx bchain.EVMNewTxSubscriber + newTxSubscription bchain.EVMClientSubscription + ChainConfig *Configuration + supportedStakingPools []string + stakingPoolNames []string + stakingPoolContracts []string } // ProcessInternalTransactions specifies if internal transactions are processed @@ -97,6 +98,8 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification ChainConfig: &c, } + // use debug_trace for internal data by default + s.GetInternalDataForBlock = s.getInternalDataForBlock ProcessInternalTransactions = c.ProcessInternalTransactions // always create parser @@ -604,7 +607,8 @@ type rpcTraceResult struct { Result rpcCallTrace `json:"result"` } -func (b *EthereumRPC) getCreationContractInfo(contract string, height uint32) *bchain.ContractInfo { +// GetCreationContractInfo retrieves the info for a contract address and sets the creation block height +func (b *EthereumRPC) GetCreationContractInfo(contract string, height uint32) *bchain.ContractInfo { ci, err := b.fetchContractInfo(contract) if ci == nil || err != nil { ci = &bchain.ContractInfo{ @@ -625,7 +629,7 @@ func (b *EthereumRPC) processCallTrace(call *rpcCallTrace, d *bchain.EthereumInt From: call.From, To: call.To, // new contract address }) - contracts = append(contracts, *b.getCreationContractInfo(call.To, blockHeight)) + contracts = append(contracts, *b.GetCreationContractInfo(call.To, blockHeight)) } else if call.Type == "SELFDESTRUCT" { d.Transfers = append(d.Transfers, bchain.EthereumInternalTransfer{ Type: bchain.SELFDESTRUCT, @@ -696,7 +700,7 @@ func (b *EthereumRPC) getInternalDataForBlock(blockHash string, blockHeight uint if r.Type == "CREATE" || r.Type == "CREATE2" { d.Type = bchain.CREATE d.Contract = r.To - contracts = append(contracts, *b.getCreationContractInfo(d.Contract, blockHeight)) + contracts = append(contracts, *b.GetCreationContractInfo(d.Contract, blockHeight)) } else if r.Type == "SELFDESTRUCT" { d.Type = bchain.SELFDESTRUCT } @@ -755,7 +759,7 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } // error fetching internal data does not stop the block processing var blockSpecificData *bchain.EthereumBlockSpecificData - internalData, contracts, err := b.getInternalDataForBlock(head.Hash, bbh.Height, body.Transactions) + internalData, contracts, err := b.GetInternalDataForBlock(head.Hash, bbh.Height, body.Transactions) // pass internalData error and ENS records in blockSpecificData to be stored if err != nil || len(ens) > 0 || len(contracts) > 0 { blockSpecificData = &bchain.EthereumBlockSpecificData{} diff --git a/bchain/coins/gnosis/evm.go b/bchain/coins/gnosis/evm.go new file mode 100644 index 0000000000..4572833f59 --- /dev/null +++ b/bchain/coins/gnosis/evm.go @@ -0,0 +1,141 @@ +package gnosis + +import ( + "context" + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/juju/errors" + "github.com/trezor/blockbook/bchain" +) + +// GnosisClient wraps a client to implement the EVMClient interface +type GnosisClient struct { + *ethclient.Client + *GnosisRPCClient +} + +// HeaderByNumber returns a block header that implements the EVMHeader interface +func (c *GnosisClient) HeaderByNumber(ctx context.Context, number *big.Int) (bchain.EVMHeader, error) { + h := &Header{} + err := c.GnosisRPCClient.CallContext(ctx, h, "eth_getBlockByNumber", ToBlockNumArg(number), false) + if err != nil { + return nil, errors.Annotatef(err, "number %v", number) + } + return &GnosisHeader{Header: h}, nil +} + +// EstimateGas returns the current estimated gas cost for executing a transaction +func (c *GnosisClient) EstimateGas(ctx context.Context, msg interface{}) (uint64, error) { + return c.Client.EstimateGas(ctx, msg.(ethereum.CallMsg)) +} + +// BalanceAt returns the balance for the given account at a specific block, or latest known block if no block number is provided +func (c *GnosisClient) BalanceAt(ctx context.Context, addrDesc bchain.AddressDescriptor, blockNumber *big.Int) (*big.Int, error) { + return c.Client.BalanceAt(ctx, common.BytesToAddress(addrDesc), blockNumber) +} + +// NonceAt returns the nonce for the given account at a specific block, or latest known block if no block number is provided +func (c *GnosisClient) NonceAt(ctx context.Context, addrDesc bchain.AddressDescriptor, blockNumber *big.Int) (uint64, error) { + return c.Client.NonceAt(ctx, common.BytesToAddress(addrDesc), blockNumber) +} + +// GnosisRPCClient wraps an rpc client to implement the EVMRPCClient interface +type GnosisRPCClient struct { + *rpc.Client +} + +// EthSubscribe subscribes to events and returns a client subscription that implements the EVMClientSubscription interface +func (c *GnosisRPCClient) EthSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (bchain.EVMClientSubscription, error) { + sub, err := c.Client.EthSubscribe(ctx, channel, args...) + if err != nil { + return nil, err + } + + return &GnosisClientSubscription{ClientSubscription: sub}, nil +} + +// GnosisHeader wraps a block header to implement the EVMHeader interface +type GnosisHeader struct { + *Header +} + +// Hash returns the block hash as a hex string +func (h *GnosisHeader) Hash() string { + return h.Header.Hash().Hex() +} + +// Number returns the block number +func (h *GnosisHeader) Number() *big.Int { + return h.Header.Number +} + +// Difficulty returns the block difficulty +func (h *GnosisHeader) Difficulty() *big.Int { + return h.Header.Difficulty +} + +// GnosisHash wraps a transaction hash to implement the EVMHash interface +type GnosisHash struct { + common.Hash +} + +// GnosisClientSubscription wraps a client subcription to implement the EVMClientSubscription interface +type GnosisClientSubscription struct { + *rpc.ClientSubscription +} + +// GnosisNewBlock wraps a block header channel to implement the EVMNewBlockSubscriber interface +type GnosisNewBlock struct { + channel chan *Header +} + +// NewGnosisNewBlock returns an initialized GnosisNewBlock struct +func NewGnosisNewBlock() *GnosisNewBlock { + return &GnosisNewBlock{channel: make(chan *Header)} +} + +// Channel returns the underlying channel as an empty interface +func (s *GnosisNewBlock) Channel() interface{} { + return s.channel +} + +// Read from the underlying channel and return a block header that implements the EVMHeader interface +func (s *GnosisNewBlock) Read() (bchain.EVMHeader, bool) { + h, ok := <-s.channel + return &GnosisHeader{Header: h}, ok +} + +// Close the underlying channel +func (s *GnosisNewBlock) Close() { + close(s.channel) +} + +// GnosisNewTx wraps a transaction hash channel to implement the EVMNewTxSubscriber interface +type GnosisNewTx struct { + channel chan common.Hash +} + +// NewGnosisNewTx returns an initialized GnosisNewTx struct +func NewGnosisNewTx() *GnosisNewTx { + return &GnosisNewTx{channel: make(chan common.Hash)} +} + +// Channel returns the underlying channel as an empty interface +func (s *GnosisNewTx) Channel() interface{} { + return s.channel +} + +// Read from the underlying channel and return a transaction hash that implements the EVMHash interface +func (s *GnosisNewTx) Read() (bchain.EVMHash, bool) { + h, ok := <-s.channel + return &GnosisHash{Hash: h}, ok +} + +// Close the underlying channel +func (s *GnosisNewTx) Close() { + close(s.channel) +} diff --git a/bchain/coins/gnosis/gnosisrpc.go b/bchain/coins/gnosis/gnosisrpc.go new file mode 100644 index 0000000000..d2d6ffcd2a --- /dev/null +++ b/bchain/coins/gnosis/gnosisrpc.go @@ -0,0 +1,203 @@ +package gnosis + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/trezor/blockbook/bchain" + "github.com/trezor/blockbook/bchain/coins/eth" +) + +const ( + // MainNet is production network + MainNet eth.Network = 100 +) + +// GnosisRPC is an interface to JSON-RPC bsc service. +type GnosisRPC struct { + *eth.EthereumRPC +} + +// NewGnosisRPC returns new GnosisRPC instance. +func NewGnosisRPC(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) { + c, err := eth.NewEthereumRPC(config, pushHandler) + if err != nil { + return nil, err + } + + s := &GnosisRPC{ + EthereumRPC: c.(*eth.EthereumRPC), + } + + // use trace_block for internal data (debug_trace memory overhead is too expensive) + c.(*eth.EthereumRPC).GetInternalDataForBlock = s.getInternalDataForBlock + + return s, nil +} + +// Initialize GnosisRPC interface +func (b *GnosisRPC) Initialize() error { + b.OpenRPC = func(url string) (bchain.EVMRPCClient, bchain.EVMClient, error) { + r, err := rpc.Dial(url) + if err != nil { + return nil, nil, err + } + rc := &GnosisRPCClient{Client: r} + ec := &GnosisClient{Client: ethclient.NewClient(r), GnosisRPCClient: rc} + return rc, ec, nil + } + + rc, ec, err := b.OpenRPC(b.ChainConfig.RPCURL) + if err != nil { + return err + } + + // set chain specific + b.Client = ec + b.RPC = rc + b.MainNetChainID = MainNet + b.NewBlock = &GnosisNewBlock{channel: make(chan *Header)} + b.NewTx = &GnosisNewTx{channel: make(chan common.Hash)} + + ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) + defer cancel() + + id, err := b.Client.NetworkID(ctx) + if err != nil { + return err + } + + // parameters for getInfo request + switch eth.Network(id.Uint64()) { + case MainNet: + b.Testnet = false + b.Network = "livenet" + default: + return errors.Errorf("Unknown network id %v", id) + } + + glog.Info("rpc: block chain ", b.Network) + + return nil +} + +type action struct { + Author string `json:"author"` + CallType string `json:"callType"` + From string `json:"from"` + Gas string `json:"gas"` + Input string `json:"input"` + RewardType string `json:"rewardType"` + To string `json:"to"` + Value string `json:"value"` +} + +type traceBlockResult struct { + Action action `json:"action"` + BlockHash string `json:"blockHash"` + BlockNumber int `json:"blockNumber"` + Error string `json:"error"` + Result struct { + GasUsed string `json:"gasUsed"` + Output string `json:"output"` + } `json:"result"` + Subtraces int `json:"subtraces"` + TraceAddress []int `json:"traceAddress"` + TransactionHash string `json:"transactionHash"` + TransactionPosition int `json:"transactionPosition"` + Type string `json:"type"` +} + +// getInternalDataForBlock extracts internal transfers and creation or destruction of contracts using the parity trace module +func (b *GnosisRPC) getInternalDataForBlock(blockHash string, blockHeight uint32, transactions []bchain.RpcTransaction) ([]bchain.EthereumInternalData, []bchain.ContractInfo, error) { + data := make([]bchain.EthereumInternalData, len(transactions)) + contracts := make([]bchain.ContractInfo, 0) + if b.EthereumRPC.ChainConfig.ProcessInternalTransactions { + var n big.Int + n.SetUint64(uint64(blockHeight)) + ctx, cancel := context.WithTimeout(context.Background(), b.Timeout) + defer cancel() + var trace []traceBlockResult + err := b.RPC.CallContext(ctx, &trace, "trace_block", ToBlockNumArg(&n)) + if err != nil { + glog.Error("trace_block ", blockHash, ", error ", err) + return data, contracts, err + } + for _, t := range trace { + // initiating call does not have any trace addresses and is not an internal transfer + if len(t.TraceAddress) == 0 { + continue + } + d := &data[t.TransactionPosition] + action := t.Action + callType := strings.ToUpper(action.CallType) + traceType := strings.ToUpper(t.Type) + value, err := hexutil.DecodeBig(action.Value) + if traceType == "CREATE" || traceType == "CREATE2" { + d.Type = bchain.CREATE + d.Contract = action.To + d.Transfers = append(d.Transfers, bchain.EthereumInternalTransfer{ + Type: bchain.CREATE, + Value: *value, + From: action.From, + To: action.To, // new contract address + }) + contracts = append(contracts, *b.GetCreationContractInfo(d.Contract, blockHeight)) + } else if t.Type == "SELFDESTRUCT" { + d.Type = bchain.SELFDESTRUCT + d.Transfers = append(d.Transfers, bchain.EthereumInternalTransfer{ + Type: bchain.SELFDESTRUCT, + Value: *value, + From: action.From, // destroyed contract address + To: action.To, + }) + contracts = append(contracts, bchain.ContractInfo{Contract: action.From, DestructedInBlock: blockHeight}) + } else if callType == "DELEGATECALL" { + // ignore DELEGATECALL (geth v1.11 the changed tracer behavior) + // https://github.com/ethereum/go-ethereum/issues/26726 + } else if t.Type == "REWARD" { + // ignore REWARD as block rewards are not associated with any specific transaction + } else if err == nil && (value.BitLen() > 0 || b.ChainConfig.ProcessZeroInternalTransactions) { + d.Transfers = append(d.Transfers, bchain.EthereumInternalTransfer{ + Value: *value, + From: action.From, + To: action.To, + }) + } + if t.Error != "" { + e := eth.PackInternalTransactionError(t.Error) + if len(e) > 1 { + d.Error = strings.ToUpper(e[:1]) + e[1:] + ". " + } + } + } + } + return data, contracts, nil +} + +// ToBlockNumArg converts a big.Int to an appropriate string representation of the number if possible +// - valid return values: (hex string, "latest", "pending", "earliest", "finalized", or "safe") +// - invalid return value: "invalid" +func ToBlockNumArg(number *big.Int) string { + if number == nil { + return "latest" + } + if number.Sign() >= 0 { + return hexutil.EncodeBig(number) + } + // It's negative. + if number.IsInt64() { + return rpc.BlockNumber(number.Int64()).String() + } + // It's negative and large, which is invalid. + return fmt.Sprintf("", number) +} diff --git a/bchain/coins/gnosis/types.go b/bchain/coins/gnosis/types.go new file mode 100644 index 0000000000..b26d5b5a05 --- /dev/null +++ b/bchain/coins/gnosis/types.go @@ -0,0 +1,349 @@ +package gnosis + +import ( + "encoding/json" + "errors" + "io" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" +) + +var hasherPool = sync.Pool{ + New: func() interface{} { return sha3.NewLegacyKeccak256() }, +} + +// rlpHash encodes x and hashes the encoded bytes +func rlpHash(x interface{}) (h common.Hash) { + sha := hasherPool.Get().(crypto.KeccakState) + defer hasherPool.Put(sha) + sha.Reset() + _ = rlp.Encode(sha, x) + _, _ = sha.Read(h[:]) + return h +} + +// Header represents a block header in the Gnosis blockchain +type Header struct { + ParentHash common.Hash `json:"parentHash"` + UncleHash common.Hash `json:"sha3Uncles"` + Coinbase common.Address `json:"miner"` + Root common.Hash `json:"stateRoot"` + TxHash common.Hash `json:"transactionsRoot"` + ReceiptHash common.Hash `json:"receiptsRoot"` + Bloom types.Bloom `json:"logsBloom"` + Difficulty *big.Int `json:"difficulty"` + Number *big.Int `json:"number"` + GasLimit uint64 `json:"gasLimit"` + GasUsed uint64 `json:"gasUsed"` + Time uint64 `json:"timestamp"` + Extra []byte `json:"extraData"` + + MixDigest *common.Hash `json:"mixHash" rlp:"optional"` + Nonce *types.BlockNonce `json:"nonce" rlp:"optional"` + + // AuRa POA Consensus + AuraStep *big.Int `json:"step" rlp:"optional"` + AuraSignature []byte `json:"signature" rlp:"optional"` + + // BaseFee was added by EIP-1559 and is ignored in legacy headers + BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"` + + // WithdrawalsHash was added by EIP-4895 and is ignored in legacy headers + WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` + + // BlobGasUsed was added by EIP-4844 and is ignored in legacy headers. + BlobGasUsed *uint64 `json:"blobGasUsed" rlp:"optional"` + + // ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. + ExcessBlobGas *uint64 `json:"excessBlobGas" rlp:"optional"` + + // ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` +} + +// MarshalJSON marshals as JSON +func (h Header) MarshalJSON() ([]byte, error) { + type Header struct { + ParentHash common.Hash `json:"parentHash"` + UncleHash common.Hash `json:"sha3Uncles"` + Coinbase common.Address `json:"miner"` + Root common.Hash `json:"stateRoot"` + TxHash common.Hash `json:"transactionsRoot"` + ReceiptHash common.Hash `json:"receiptsRoot"` + Bloom types.Bloom `json:"logsBloom"` + Difficulty *hexutil.Big `json:"difficulty"` + Number *hexutil.Big `json:"number"` + GasLimit hexutil.Uint64 `json:"gasLimit"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + Time hexutil.Uint64 `json:"timestamp"` + Extra hexutil.Bytes `json:"extraData"` + MixDigest *common.Hash `json:"mixHash" rlp:"optional"` + Nonce *types.BlockNonce `json:"nonce" rlp:"optional"` + AuraStep *uint64 `json:"step" rlp:"optional"` + AuraSignature hexutil.Bytes `json:"signature" rlp:"optional"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + Hash common.Hash `json:"hash"` + } + var enc Header + enc.ParentHash = h.ParentHash + enc.UncleHash = h.UncleHash + enc.Coinbase = h.Coinbase + enc.Root = h.Root + enc.TxHash = h.TxHash + enc.ReceiptHash = h.ReceiptHash + enc.Bloom = h.Bloom + enc.Difficulty = (*hexutil.Big)(h.Difficulty) + enc.Number = (*hexutil.Big)(h.Number) + enc.GasLimit = hexutil.Uint64(h.GasLimit) + enc.GasUsed = hexutil.Uint64(h.GasUsed) + enc.Time = hexutil.Uint64(h.Time) + enc.Extra = h.Extra + enc.MixDigest = h.MixDigest + enc.Nonce = h.Nonce + auraStep := h.AuraStep.Uint64() + enc.AuraStep = &auraStep + enc.AuraSignature = h.AuraSignature + enc.BaseFee = (*hexutil.Big)(h.BaseFee) + enc.WithdrawalsHash = h.WithdrawalsHash + enc.BlobGasUsed = (*hexutil.Uint64)(h.BlobGasUsed) + enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas) + enc.ParentBeaconRoot = h.ParentBeaconRoot + enc.Hash = h.Hash() + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON +func (h *Header) UnmarshalJSON(input []byte) error { + type Header struct { + ParentHash *common.Hash `json:"parentHash"` + UncleHash *common.Hash `json:"sha3Uncles"` + Coinbase *common.Address `json:"miner"` + Root *common.Hash `json:"stateRoot"` + TxHash *common.Hash `json:"transactionsRoot"` + ReceiptHash *common.Hash `json:"receiptsRoot"` + Bloom *types.Bloom `json:"logsBloom"` + Difficulty *hexutil.Big `json:"difficulty"` + Number *hexutil.Big `json:"number"` + GasLimit *hexutil.Uint64 `json:"gasLimit"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + Time *hexutil.Uint64 `json:"timestamp"` + Extra *hexutil.Bytes `json:"extraData"` + MixDigest *common.Hash `json:"mixHash" rlp:"optional"` + Nonce *types.BlockNonce `json:"nonce" rlp:"optional"` + AuraStep *uint64 `json:"step" rlp:"optional"` + AuraSignature *hexutil.Bytes `json:"signature" rlp:"optional"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + } + var dec Header + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ParentHash == nil { + return errors.New("missing required field 'parentHash' for Header") + } + h.ParentHash = *dec.ParentHash + if dec.UncleHash == nil { + return errors.New("missing required field 'sha3Uncles' for Header") + } + h.UncleHash = *dec.UncleHash + if dec.Coinbase != nil { + h.Coinbase = *dec.Coinbase + } + if dec.Root == nil { + return errors.New("missing required field 'stateRoot' for Header") + } + h.Root = *dec.Root + if dec.TxHash == nil { + return errors.New("missing required field 'transactionsRoot' for Header") + } + h.TxHash = *dec.TxHash + if dec.ReceiptHash == nil { + return errors.New("missing required field 'receiptsRoot' for Header") + } + h.ReceiptHash = *dec.ReceiptHash + if dec.Bloom == nil { + return errors.New("missing required field 'logsBloom' for Header") + } + h.Bloom = *dec.Bloom + if dec.Difficulty == nil { + return errors.New("missing required field 'difficulty' for Header") + } + h.Difficulty = (*big.Int)(dec.Difficulty) + if dec.Number == nil { + return errors.New("missing required field 'number' for Header") + } + h.Number = (*big.Int)(dec.Number) + if dec.GasLimit == nil { + return errors.New("missing required field 'gasLimit' for Header") + } + h.GasLimit = uint64(*dec.GasLimit) + if dec.GasUsed == nil { + return errors.New("missing required field 'gasUsed' for Header") + } + h.GasUsed = uint64(*dec.GasUsed) + if dec.Time == nil { + return errors.New("missing required field 'timestamp' for Header") + } + h.Time = uint64(*dec.Time) + if dec.Extra == nil { + return errors.New("missing required field 'extraData' for Header") + } + h.Extra = *dec.Extra + if dec.AuraSignature != nil { + if dec.AuraStep == nil { + return errors.New("missing required field 'step' for Header") + } + h.AuraStep = new(big.Int).SetUint64(*dec.AuraStep) + h.AuraSignature = *dec.AuraSignature + } else { + if dec.MixDigest == nil { + return errors.New("missing required field 'mixHash' for Header") + } + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' for Header") + } + h.MixDigest = dec.MixDigest + h.Nonce = dec.Nonce + } + if dec.BaseFee != nil { + h.BaseFee = (*big.Int)(dec.BaseFee) + } + if dec.WithdrawalsHash != nil { + h.WithdrawalsHash = dec.WithdrawalsHash + } + if dec.BlobGasUsed != nil { + h.BlobGasUsed = (*uint64)(dec.BlobGasUsed) + } + if dec.ExcessBlobGas != nil { + h.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) + } + if dec.ParentBeaconRoot != nil { + h.ParentBeaconRoot = dec.ParentBeaconRoot + } + return nil +} + +// EncodeRLP encodes as RLP +func (h *Header) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteBytes(h.ParentHash[:]) + w.WriteBytes(h.UncleHash[:]) + w.WriteBytes(h.Coinbase[:]) + w.WriteBytes(h.Root[:]) + w.WriteBytes(h.TxHash[:]) + w.WriteBytes(h.ReceiptHash[:]) + w.WriteBytes(h.Bloom[:]) + if h.Difficulty == nil { + _, _ = w.Write(rlp.EmptyString) + } else { + if h.Difficulty.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(h.Difficulty) + } + if h.Number == nil { + _, _ = w.Write(rlp.EmptyString) + } else { + if h.Number.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(h.Number) + } + w.WriteUint64(h.GasLimit) + w.WriteUint64(h.GasUsed) + w.WriteUint64(h.Time) + w.WriteBytes(h.Extra) + _tmp1 := len(h.AuraSignature) > 0 + _tmp2 := h.AuraStep != nil + if _tmp1 || _tmp2 { + if h.AuraStep == nil { + _, _ = w.Write(rlp.EmptyString) + } else { + if h.AuraStep.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(h.AuraStep) + } + w.WriteBytes(h.AuraSignature[:]) + } + _tmp3 := h.MixDigest != nil + _tmp4 := h.Nonce != nil + if _tmp3 || _tmp4 { + if h.MixDigest == nil { + _, _ = w.Write([]byte{0x80}) + } else { + w.WriteBytes(h.MixDigest[:]) + } + if h.Nonce == nil { + _, _ = w.Write(rlp.EmptyString) + } else { + w.WriteBytes(h.Nonce[:]) + } + } + _tmp5 := h.BaseFee != nil + _tmp6 := h.WithdrawalsHash != nil + _tmp7 := h.BlobGasUsed != nil + _tmp8 := h.ExcessBlobGas != nil + _tmp9 := h.ParentBeaconRoot != nil + if _tmp5 || _tmp6 || _tmp7 || _tmp8 || _tmp9 { + if h.BaseFee == nil { + _, _ = w.Write(rlp.EmptyString) + } else { + if h.BaseFee.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(h.BaseFee) + } + } + if _tmp6 || _tmp7 || _tmp8 || _tmp9 { + if h.WithdrawalsHash == nil { + _, _ = w.Write([]byte{0x80}) + } else { + w.WriteBytes(h.WithdrawalsHash[:]) + } + } + if _tmp7 || _tmp8 || _tmp9 { + if h.BlobGasUsed == nil { + _, _ = w.Write([]byte{0x80}) + } else { + w.WriteUint64((*h.BlobGasUsed)) + } + } + if _tmp8 || _tmp9 { + if h.ExcessBlobGas == nil { + _, _ = w.Write([]byte{0x80}) + } else { + w.WriteUint64((*h.ExcessBlobGas)) + } + } + if _tmp9 { + if h.ParentBeaconRoot == nil { + _, _ = w.Write([]byte{0x80}) + } else { + w.WriteBytes(h.ParentBeaconRoot[:]) + } + } + w.ListEnd(_tmp0) + return w.Flush() +} + +// Hash returns the block hash of the header, which is simply the keccak256 hash of its RLP encoding +func (h *Header) Hash() common.Hash { + return rlpHash(h) +} diff --git a/blockbook.go b/blockbook.go index a0065ec680..155b085553 100644 --- a/blockbook.go +++ b/blockbook.go @@ -43,9 +43,10 @@ const exitCodeFatal = 255 var ( configFile = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") - dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") - dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb") + dbMaxAddrContracts = flag.Int("dbmaxaddrcontracts", 1<<20, "max size of the address contracts map") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -169,7 +170,7 @@ func mainWithExitCode() int { return exitCodeFatal } - index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics, *extendedIndex) + index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics, *extendedIndex, *dbMaxAddrContracts) if err != nil { glog.Error("rocksDB: ", err) return exitCodeFatal diff --git a/configs/coins/gnosis.json b/configs/coins/gnosis.json new file mode 100644 index 0000000000..842dd16841 --- /dev/null +++ b/configs/coins/gnosis.json @@ -0,0 +1,72 @@ +{ + "coin": { + "name": "Gnosis", + "shortcut": "xDAI", + "label": "Gnosis", + "alias": "gnosis" + }, + "ports": { + "backend_rpc": 8074, + "backend_p2p": 38374, + "backend_http": 8174, + "backend_authrpc": 8574, + "blockbook_internal": 9074, + "blockbook_public": 9174 + }, + "ipc": { + "rpc_url_template": "ws://127.0.0.1:{{.Ports.BackendRPC}}", + "rpc_timeout": 25 + }, + "backend": { + "package_name": "backend-gnosis", + "package_revision": "satoshilabs-1", + "system_user": "gnosis", + "version": "1.25.1", + "binary_url": "https://github.com/NethermindEth/nethermind/releases/download/1.25.1/nethermind-1.25.1-e37c10db-linux-x64.zip", + "verification_type": "sha256", + "verification_source": "f006c47418e61db230637e2cfe95b4fee1de1578195bc2699061b7a9d7fc5fc8", + "extract_command": "unzip -d backend", + "exclude_files": [], + "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/Nethermind.Runner --config gnosis --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --Network.P2PPort {{.Ports.BackendP2P}} --JsonRpc.Port {{.Ports.BackendHttp}} --JsonRpc.EnabledModules \"eth,net,web3,trace,subscribe,txpool,rpc\" --Init.WebSocketsEnabled true --JsonRpc.WebSocketsPort {{.Ports.BackendRPC}} --JsonRpc.EnginePort {{.Ports.BackendAuthRpc}} --JsonRpc.JwtSecretFile {{.Env.BackendDataPath}}/gnosis_consensus/backend/beacon/jwt.hex 2>&1 >> {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", + "postinst_script_template": "", + "service_type": "simple", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": true, + "server_config_file": "", + "client_config_file": "", + "platforms": { + "arm64": { + "binary_url": "https://github.com/NethermindEth/nethermind/releases/download/1.25.1/nethermind-1.25.1-e37c10db-linux-arm64.zip", + "verification_source": "18372c26c97ad47dabc0fa4c01015bc837798f65148cfe52cae67bcdda70dbc2" + } + } + }, + "blockbook": { + "package_name": "blockbook-gnosis", + "system_user": "blockbook-gnosis", + "internal_binding_template": ":{{.Ports.BlockbookInternal}}", + "public_binding_template": ":{{.Ports.BlockbookPublic}}", + "explorer_url": "", + "additional_params": "", + "block_chain": { + "parse": true, + "mempool_workers": 8, + "mempool_sub_workers": 2, + "block_addresses_to_keep": 300, + "additional_params": { + "consensusNodeVersion": "http://localhost:8075/eth/v1/node/version", + "mempoolTxTimeoutHours": 48, + "queryBackendOnMempoolResync": false, + "fiat_rates": "coingecko", + "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", + "fiat_rates_params": "{\"coin\": \"xdai\",\"platformIdentifier\": \"xdai\",\"platformVsCurrency\": \"usd\",\"periodSeconds\": 900}" + } + } + }, + "meta": { + "package_maintainer": "IT", + "package_maintainer_email": "it@satoshilabs.com" + } +} diff --git a/configs/coins/gnosis_archive.json b/configs/coins/gnosis_archive.json new file mode 100644 index 0000000000..6ae94bbaae --- /dev/null +++ b/configs/coins/gnosis_archive.json @@ -0,0 +1,75 @@ +{ + "coin": { + "name": "Gnosis Archive", + "shortcut": "xDAI", + "label": "Gnosis", + "alias": "gnosis_archive" + }, + "ports": { + "backend_rpc": 8076, + "backend_p2p": 38376, + "backend_http": 8176, + "backend_authrpc": 8576, + "blockbook_internal": 9076, + "blockbook_public": 9176 + }, + "ipc": { + "rpc_url_template": "ws://127.0.0.1:{{.Ports.BackendRPC}}", + "rpc_timeout": 25 + }, + "backend": { + "package_name": "backend-gnosis-archive", + "package_revision": "satoshilabs-1", + "system_user": "gnosis", + "version": "1.25.1", + "binary_url": "https://github.com/NethermindEth/nethermind/releases/download/1.25.1/nethermind-1.25.1-e37c10db-linux-x64.zip", + "verification_type": "sha256", + "verification_source": "f006c47418e61db230637e2cfe95b4fee1de1578195bc2699061b7a9d7fc5fc8", + "extract_command": "unzip -d backend", + "exclude_files": [], + "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/Nethermind.Runner --config gnosis --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --Network.P2PPort {{.Ports.BackendP2P}} --JsonRpc.Port {{.Ports.BackendHttp}} --JsonRpc.EnabledModules \"eth,net,web3,trace,subscribe,txpool,rpc\" --Init.WebSocketsEnabled true --JsonRpc.WebSocketsPort {{.Ports.BackendRPC}} --JsonRpc.EnginePort {{.Ports.BackendAuthRpc}} --JsonRpc.JwtSecretFile {{.Env.BackendDataPath}}/gnosis_consensus/backend/beacon/jwt.hex 2>&1 >> {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", + "postinst_script_template": "", + "service_type": "simple", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": true, + "server_config_file": "", + "client_config_file": "", + "platforms": { + "arm64": { + "binary_url": "https://github.com/NethermindEth/nethermind/releases/download/1.25.1/nethermind-1.25.1-e37c10db-linux-arm64.zip", + "verification_source": "18372c26c97ad47dabc0fa4c01015bc837798f65148cfe52cae67bcdda70dbc2" + } + } + }, + "blockbook": { + "package_name": "blockbook-gnosis-archive", + "system_user": "blockbook-gnosis", + "internal_binding_template": ":{{.Ports.BlockbookInternal}}", + "public_binding_template": ":{{.Ports.BlockbookPublic}}", + "explorer_url": "", + "additional_params": "-workers=16", + "block_chain": { + "parse": true, + "mempool_workers": 8, + "mempool_sub_workers": 2, + "block_addresses_to_keep": 600, + "additional_params": { + "consensusNodeVersion": "http://localhost:8077/eth/v1/node/version", + "address_aliases": true, + "mempoolTxTimeoutHours": 48, + "processInternalTransactions": true, + "queryBackendOnMempoolResync": false, + "fiat_rates": "coingecko", + "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", + "fiat_rates_params": "{\"coin\": \"xdai\",\"platformIdentifier\": \"xdai\",\"platformVsCurrency\": \"usd\",\"periodSeconds\": 900}", + "fourByteSignatures": "https://www.4byte.directory/api/v1/signatures/" + } + } + }, + "meta": { + "package_maintainer": "IT", + "package_maintainer_email": "it@satoshilabs.com" + } +} diff --git a/configs/coins/gnosis_archive_consensus.json b/configs/coins/gnosis_archive_consensus.json new file mode 100644 index 0000000000..200fd35dc4 --- /dev/null +++ b/configs/coins/gnosis_archive_consensus.json @@ -0,0 +1,38 @@ +{ + "coin": { + "name": "Gnosis Archive Lighthouse", + "shortcut": "xDAI", + "label": "Gnosis", + "alias": "gnosis_archive_lighthouse" + }, + "ports": { + "backend_p2p": 38377, + "backend_rpc": 8077, + "blockbook_internal": 9077, + "blockbook_public": 9177 + }, + "backend": { + "package_name": "backend-gnosis-archive-lighthouse", + "package_revision": "satoshilabs-1", + "system_user": "gnosis", + "version": "4.5.0", + "binary_url": "https://github.com/sigp/lighthouse/releases/download/v4.5.0/lighthouse-v4.5.0-x86_64-unknown-linux-gnu.tar.gz", + "verification_type": "gpg", + "verification_source": "https://github.com/sigp/lighthouse/releases/download/v4.5.0/lighthouse-v4.5.0-x86_64-unknown-linux-gnu.tar.gz.asc", + "extract_command": "tar -C backend -xf", + "exclude_files": [], + "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/lighthouse beacon_node --network gnosis --disable-upnp --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port {{.Ports.BackendP2P}} --http --http-port {{.Ports.BackendRPC}} --execution-endpoint http://127.0.0.1:8576 --checkpoint-sync-url https://checkpoint.gnosischain.com/ --execution-jwt-secret-key ba31ba13b58d8169af0a490ae464b472c5d6a8c38b5cbe6a7a5c338ba70e624f 2>> {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", + "postinst_script_template": "", + "service_type": "simple", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": false, + "server_config_file": "", + "client_config_file": "" + }, + "meta": { + "package_maintainer": "IT", + "package_maintainer_email": "it@satoshilabs.com" + } +} diff --git a/configs/coins/gnosis_consensus.json b/configs/coins/gnosis_consensus.json new file mode 100644 index 0000000000..1f28971dc7 --- /dev/null +++ b/configs/coins/gnosis_consensus.json @@ -0,0 +1,38 @@ +{ + "coin": { + "name": "Gnosis Lighthouse", + "shortcut": "xDAI", + "label": "Gnosis", + "alias": "gnosis_lighthouse" + }, + "ports": { + "backend_p2p": 38375, + "backend_rpc": 8075, + "blockbook_internal": 9075, + "blockbook_public": 9175 + }, + "backend": { + "package_name": "backend-gnosis-lighthouse", + "package_revision": "satoshilabs-1", + "system_user": "gnosis", + "version": "4.5.0", + "binary_url": "https://github.com/sigp/lighthouse/releases/download/v4.5.0/lighthouse-v4.5.0-x86_64-unknown-linux-gnu.tar.gz", + "verification_type": "gpg", + "verification_source": "https://github.com/sigp/lighthouse/releases/download/v4.5.0/lighthouse-v4.5.0-x86_64-unknown-linux-gnu.tar.gz.asc", + "extract_command": "tar -C backend -xf", + "exclude_files": [], + "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/lighthouse beacon_node --network gnosis --disable-upnp --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port {{.Ports.BackendP2P}} --http --http-port {{.Ports.BackendRPC}} --execution-endpoint http://127.0.0.1:8574 --checkpoint-sync-url https://checkpoint.gnosischain.com/ --execution-jwt-secret-key ba31ba13b58d8169af0a490ae464b472c5d6a8c38b5cbe6a7a5c338ba70e624f 2>> {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", + "postinst_script_template": "", + "service_type": "simple", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": false, + "server_config_file": "", + "client_config_file": "" + }, + "meta": { + "package_maintainer": "IT", + "package_maintainer_email": "it@satoshilabs.com" + } +} diff --git a/db/bulkconnect.go b/db/bulkconnect.go index 2a044bc12a..aa382a1bea 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -29,30 +29,26 @@ type BulkConnect struct { txAddressesMap map[string]*TxAddresses blockFilters map[string][]byte balances map[string]*AddrBalance - addressContracts map[string]*AddrContracts height uint32 } const ( - maxBulkAddresses = 80000 - maxBulkTxAddresses = 500000 - partialStoreAddresses = maxBulkTxAddresses / 10 - maxBulkBalances = 700000 - partialStoreBalances = maxBulkBalances / 10 - maxBulkAddrContracts = 1200000 - partialStoreAddrContracts = maxBulkAddrContracts / 10 - maxBlockFilters = 1000 + maxBulkAddresses = 80000 + maxBulkTxAddresses = 500000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 700000 + partialStoreBalances = maxBulkBalances / 10 + maxBlockFilters = 1000 ) // InitBulkConnect initializes bulk connect and switches DB to inconsistent state func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { b := &BulkConnect{ - d: d, - chainType: d.chainParser.GetChainType(), - txAddressesMap: make(map[string]*TxAddresses), - balances: make(map[string]*AddrBalance), - addressContracts: make(map[string]*AddrContracts), - blockFilters: make(map[string][]byte), + d: d, + chainType: d.chainParser.GetChainType(), + txAddressesMap: make(map[string]*TxAddresses), + balances: make(map[string]*AddrBalance), + blockFilters: make(map[string][]byte), } if err := d.SetInconsistentState(true); err != nil { return nil, err @@ -266,15 +262,15 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs func (b *BulkConnect) storeAddressContracts(wb *grocksdb.WriteBatch, all bool) (int, error) { var ac map[string]*AddrContracts if all { - ac = b.addressContracts - b.addressContracts = make(map[string]*AddrContracts) + ac = b.d.addressContracts + b.d.addressContracts = make(map[string]*AddrContracts) } else { ac = make(map[string]*AddrContracts) // store some random address contracts - for k, a := range b.addressContracts { + for k, a := range b.d.addressContracts { ac[k] = a - delete(b.addressContracts, k) - if len(ac) >= partialStoreAddrContracts { + delete(b.d.addressContracts, k) + if len(ac) >= (b.d.maxAddrContracts / 10) { break } } @@ -299,20 +295,20 @@ func (b *BulkConnect) parallelStoreAddressContracts(c chan error, all bool) { c <- err return } - glog.Info("rocksdb: height ", b.height, ", stored ", count, " addressContracts, ", len(b.addressContracts), " remaining, done in ", time.Since(start)) + glog.Info("rocksdb: height ", b.height, ", stored ", count, " addressContracts, ", len(b.d.addressContracts), " remaining, done in ", time.Since(start)) c <- nil } func (b *BulkConnect) connectBlockEthereumType(block *bchain.Block, storeBlockTxs bool) error { addresses := make(addressesMap) - blockTxs, err := b.d.processAddressesEthereumType(block, addresses, b.addressContracts) + blockTxs, err := b.d.processAddressesEthereumType(block, addresses, b.d.addressContracts) if err != nil { return err } b.ethBlockTxs = append(b.ethBlockTxs, blockTxs...) var storeAddrContracts chan error var sa bool - if len(b.addressContracts) > maxBulkAddrContracts { + if len(b.d.addressContracts) > b.d.maxAddrContracts { sa = true storeAddrContracts = make(chan error) go b.parallelStoreAddressContracts(storeAddrContracts, false) diff --git a/db/rocksdb.go b/db/rocksdb.go index 9c5a3b2fdf..da684add8c 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "math" "math/big" "os" "path/filepath" @@ -59,18 +60,20 @@ const ( // RocksDB handle type RocksDB struct { - path string - db *grocksdb.DB - wo *grocksdb.WriteOptions - ro *grocksdb.ReadOptions - cfh []*grocksdb.ColumnFamilyHandle - chainParser bchain.BlockChainParser - is *common.InternalState - metrics *common.Metrics - cache *grocksdb.Cache - maxOpenFiles int - cbs connectBlockStats - extendedIndex bool + path string + db *grocksdb.DB + wo *grocksdb.WriteOptions + ro *grocksdb.ReadOptions + cfh []*grocksdb.ColumnFamilyHandle + chainParser bchain.BlockChainParser + is *common.InternalState + metrics *common.Metrics + cache *grocksdb.Cache + maxOpenFiles int + cbs connectBlockStats + extendedIndex bool + maxAddrContracts int + addressContracts map[string]*AddrContracts } const ( @@ -128,7 +131,7 @@ func openDB(path string, c *grocksdb.Cache, openFiles int) (*grocksdb.DB, []*gro // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics, extendedIndex bool) (d *RocksDB, err error) { +func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics, extendedIndex bool, maxAddrContracts int) (*RocksDB, error) { glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) cfNames = append([]string{}, cfBaseNames...) @@ -147,9 +150,23 @@ func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockCha if err != nil { return nil, err } - wo := grocksdb.NewDefaultWriteOptions() - ro := grocksdb.NewDefaultReadOptions() - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles, connectBlockStats{}, extendedIndex}, nil + d := &RocksDB{ + path: path, + db: db, + wo: grocksdb.NewDefaultWriteOptions(), + ro: grocksdb.NewDefaultReadOptions(), + cfh: cfh, + chainParser: parser, + is: nil, + metrics: metrics, + cache: c, + maxOpenFiles: maxOpenFiles, + cbs: connectBlockStats{}, + extendedIndex: extendedIndex, + maxAddrContracts: maxAddrContracts, + addressContracts: make(map[string]*AddrContracts), + } + return d, nil } func (d *RocksDB) closeDB() error { @@ -331,6 +348,22 @@ const ( opDelete = 1 ) +func (d *RocksDB) pruneAddressContracts(attempt float64) { + desiredSize := d.maxAddrContracts / 2 + threshold := math.Pow(10, attempt) + for k, v := range d.addressContracts { + if len(d.addressContracts) == desiredSize { + break + } + if len(v.Contracts) < int(threshold) { + delete(d.addressContracts, k) + } + } + if len(d.addressContracts) > desiredSize { + d.pruneAddressContracts(attempt + 1) + } +} + // ConnectBlock indexes addresses in the block and stores them in db func (d *RocksDB) ConnectBlock(block *bchain.Block) error { wb := grocksdb.NewWriteBatch() @@ -375,6 +408,9 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error { } } } else if chainType == bchain.ChainEthereumType { + if len(d.addressContracts) > d.maxAddrContracts { + d.pruneAddressContracts(1) + } addressContracts := make(map[string]*AddrContracts) blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts) if err != nil { diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go index 29bff516cd..fb09d757d7 100644 --- a/db/rocksdb_ethereumtype.go +++ b/db/rocksdb_ethereumtype.go @@ -10,6 +10,7 @@ import ( vlq "github.com/bsm/go-vlq" "github.com/golang/glog" + "github.com/golang/protobuf/proto" "github.com/juju/errors" "github.com/linxGnu/grocksdb" "github.com/trezor/blockbook/bchain" @@ -124,45 +125,79 @@ type AddrContracts struct { Contracts []AddrContract } -// packAddrContract packs AddrContracts into a byte buffer -func packAddrContracts(acs *AddrContracts) []byte { - buf := make([]byte, 0, 128) - varBuf := make([]byte, maxPackedBigintBytes) - l := packVaruint(acs.TotalTxs, varBuf) - buf = append(buf, varBuf[:l]...) - l = packVaruint(acs.NonContractTxs, varBuf) - buf = append(buf, varBuf[:l]...) - l = packVaruint(acs.InternalTxs, varBuf) - buf = append(buf, varBuf[:l]...) - for _, ac := range acs.Contracts { - buf = append(buf, ac.Contract...) - l = packVaruint(uint(ac.Type)+ac.Txs<<2, varBuf) - buf = append(buf, varBuf[:l]...) - if ac.Type == bchain.FungibleToken { - l = packBigint(&ac.Value, varBuf) - buf = append(buf, varBuf[:l]...) - } else if ac.Type == bchain.NonFungibleToken { - l = packVaruint(uint(len(ac.Ids)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ac.Ids { - l = packBigint(&ac.Ids[i], varBuf) - buf = append(buf, varBuf[:l]...) - } - } else { // bchain.ERC1155 - l = packVaruint(uint(len(ac.MultiTokenValues)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ac.MultiTokenValues { - l = packBigint(&ac.MultiTokenValues[i].Id, varBuf) - buf = append(buf, varBuf[:l]...) - l = packBigint(&ac.MultiTokenValues[i].Value, varBuf) - buf = append(buf, varBuf[:l]...) +// packAddrContract packs AddrContracts into a protobuf encoded byte slice +func packAddrContracts(acs *AddrContracts) ([]byte, error) { + ptContracts := make([]*eth.ProtoAddrContracts_AddrContract, len(acs.Contracts)) + for i, c := range acs.Contracts { + ptIds := make([][]byte, len(c.Ids)) + for j, id := range c.Ids { + ptIds[j] = id.Bytes() + } + ptMultiTokenValues := make([]*eth.ProtoAddrContracts_MultiTokenValue, len(c.MultiTokenValues)) + for k, mtv := range c.MultiTokenValues { + ptMultiTokenValues[k] = ð.ProtoAddrContracts_MultiTokenValue{ + Id: mtv.Id.Bytes(), + Value: mtv.Value.Bytes(), } } + ptContracts[i] = ð.ProtoAddrContracts_AddrContract{ + Contract: c.Contract, + Ids: ptIds, + MultiTokenValues: ptMultiTokenValues, + Type: int64(c.Type), + Txs: uint64(c.Txs), + Value: c.Value.Bytes(), + } } - return buf + pt := ð.ProtoAddrContracts{ + TotalTxs: uint64(acs.TotalTxs), + InternalTxs: uint64(acs.InternalTxs), + NonContractTxs: uint64(acs.NonContractTxs), + Contracts: ptContracts, + } + return proto.Marshal(pt) } +// unpackAddrContract unpacks the protobuf encoded byte slice into AddrContracts func unpackAddrContracts(buf []byte, addrDesc bchain.AddressDescriptor) (*AddrContracts, error) { + pt := ð.ProtoAddrContracts{} + err := proto.Unmarshal(buf, pt) + if err != nil { + return unpackAddrContractsLegacy(buf, addrDesc) + } + contracts := make([]AddrContract, len(pt.Contracts)) + for i, c := range pt.Contracts { + ids := make([]big.Int, len(c.Ids)) + for j, id := range c.Ids { + ids[j] = *new(big.Int).SetBytes(id) + } + multiTokenValues := make(MultiTokenValues, len(c.MultiTokenValues)) + for k, mtv := range c.MultiTokenValues { + multiTokenValues[k] = bchain.MultiTokenValue{ + Id: *new(big.Int).SetBytes(mtv.Id), + Value: *new(big.Int).SetBytes(mtv.Value), + } + } + contracts[i] = AddrContract{ + Type: bchain.TokenType(c.Type), + Contract: c.Contract, + Txs: uint(c.Txs), + Value: *new(big.Int).SetBytes(c.Value), + Ids: ids, + MultiTokenValues: multiTokenValues, + } + } + acs := &AddrContracts{ + TotalTxs: uint(pt.TotalTxs), + NonContractTxs: uint(pt.NonContractTxs), + InternalTxs: uint(pt.InternalTxs), + Contracts: contracts, + } + return acs, nil +} + +// unpackAddrContractsLegacy unpacks AddrContracts from legacy manual packed byte slice +func unpackAddrContractsLegacy(buf []byte, addrDesc bchain.AddressDescriptor) (*AddrContracts, error) { tt, l := unpackVaruint(buf) buf = buf[l:] nct, l := unpackVaruint(buf) @@ -226,7 +261,10 @@ func (d *RocksDB) storeAddressContracts(wb *grocksdb.WriteBatch, acm map[string] if acs == nil || (acs.NonContractTxs == 0 && acs.InternalTxs == 0 && len(acs.Contracts) == 0) { wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc)) } else { - buf := packAddrContracts(acs) + buf, err := packAddrContracts(acs) + if err != nil { + return err + } wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf) } } @@ -338,7 +376,7 @@ func addToContract(c *AddrContract, contractIndex int, index int32, contract bch func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, transfer *bchain.TokenTransfer, addTxCount bool, addresses addressesMap, addressContracts map[string]*AddrContracts) error { var err error strAddrDesc := string(addrDesc) - ac, e := addressContracts[strAddrDesc] + ac, e := d.addressContracts[strAddrDesc] if !e { ac, err = d.GetAddrDescContracts(addrDesc) if err != nil { @@ -348,8 +386,10 @@ func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.Address ac = &AddrContracts{} } addressContracts[strAddrDesc] = ac + d.addressContracts[strAddrDesc] = ac d.cbs.balancesMiss++ } else { + addressContracts[strAddrDesc] = ac d.cbs.balancesHit++ } if contract == nil { @@ -1438,7 +1478,10 @@ func (d *RocksDB) SortAddressContracts(stop chan os.Signal) error { if err := func() error { wb := grocksdb.NewWriteBatch() defer wb.Destroy() - buf := packAddrContracts(ca) + buf, err := packAddrContracts(ca) + if err != nil { + return err + } wb.PutCF(d.cfh[cfAddressContracts], addrDesc, buf) return d.WriteBatch(wb) }(); err != nil { diff --git a/docs/ports.md b/docs/ports.md index 212d4ace6b..a4f4c3e7ff 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -1,82 +1,84 @@ # Registry of ports -| coin | blockbook public | blockbook internal | backend rpc | backend service ports (zmq) | -|----------------------------------|------------------|--------------------|-------------|--------------------------------------| -| Ethereum Archive | 9116 | 9016 | 8016 | 38316 p2p, 8116 http, 8516 authrpc | -| Bitcoin | 9130 | 9030 | 8030 | 38330 | -| Bitcoin Cash | 9131 | 9031 | 8031 | 38331 | -| Zcash | 9132 | 9032 | 8032 | 38332 | -| Dash | 9133 | 9033 | 8033 | 38333 | -| Litecoin | 9134 | 9034 | 8034 | 38334 | -| Bitcoin Gold | 9135 | 9035 | 8035 | 38335 | -| Ethereum | 9136 | 9036 | 8036 | 38336 p2p, 8136 http, 8536 authrpc | -| Ethereum Classic | 9137 | 9037 | 8037 | 38337 p2p, 8137 http | -| Dogecoin | 9138 | 9038 | 8038 | 38338 | -| Namecoin | 9139 | 9039 | 8039 | 38339 | -| Vertcoin | 9140 | 9040 | 8040 | 38340 | -| Monacoin | 9141 | 9041 | 8041 | 38341 | -| DigiByte | 9142 | 9042 | 8042 | 38342 | -| Myriad | 9143 | 9043 | 8043 | 38343 | -| GameCredits | 9144 | 9044 | 8044 | 38344 | -| Groestlcoin | 9145 | 9045 | 8045 | 38345 | -| Bitcoin Cash SV | 9146 | 9046 | 8046 | 38346 | -| Liquid | 9147 | 9047 | 8047 | 38347 | -| Fujicoin | 9148 | 9048 | 8048 | 38348 | -| PIVX | 9149 | 9049 | 8049 | 38349 | -| Firo | 9150 | 9050 | 8050 | 38350 | -| Koto | 9151 | 9051 | 8051 | 38351 | -| Bellcoin | 9152 | 9052 | 8052 | 38352 | -| NULS | 9153 | 9053 | 8053 | 38353 | -| Bitcore | 9154 | 9054 | 8054 | 38354 | -| Viacoin | 9155 | 9055 | 8055 | 38355 | -| VIPSTARCOIN | 9156 | 9056 | 8056 | 38356 | -| MonetaryUnit | 9157 | 9057 | 8057 | 38357 | -| Flux | 9158 | 9058 | 8058 | 38358 | -| Ravencoin | 9159 | 9059 | 8059 | 38359 | -| Ritocoin | 9160 | 9060 | 8060 | 38360 | -| Decred | 9161 | 9061 | 8061 | 38361 | -| SnowGem | 9162 | 9062 | 8062 | 38362 | -| BNB Smart Chain | 9164 | 9064 | 8064 | 38364 p2p, 8164 http | -| BNB Smart Chain Archive | 9165 | 9065 | 8065 | 38365 p2p, 8165 http | -| Flo | 9166 | 9066 | 8066 | 38366 | -| Polis | 9167 | 9067 | 8067 | 38367 | -| Polygon | 9170 | 9070 | 8070 | 38370 p2p, 8170 http | -| Polygon Archive | 9172 | 9072 | 8072 | 38372 p2p, 8172 http | -| Qtum | 9188 | 9088 | 8088 | 38388 | -| Divi Project | 9189 | 9089 | 8089 | 38389 | -| CPUchain | 9190 | 9090 | 8090 | 38390 | -| DeepOnion | 9191 | 9091 | 8091 | 38391 | -| Unobtanium | 9192 | 9092 | 65535 | 38392 | -| Omotenashicoin | 9194 | 9094 | 8094 | 38394 | -| BitZeny | 9195 | 9095 | 8095 | 38395 | -| Trezarcoin | 9196 | 9096 | 8096 | 38396 | -| eCash | 9197 | 9097 | 8097 | 38397 | -| Avalanche | 9198 | 9098 | 8098 | 38398 p2p | -| Avalanche Archive | 9199 | 9099 | 8099 | 38399 p2p | -| Ethereum Testnet Holesky | 19116 | 19016 | 18016 | 18116 http, 18516 authrpc, 48316 p2p | -| Bitcoin Signet | 19120 | 19020 | 18020 | 48320 | -| Bitcoin Regtest | 19121 | 19021 | 18021 | 48321 | -| Bitcoin Testnet | 19130 | 19030 | 18030 | 48330 | -| Bitcoin Cash Testnet | 19131 | 19031 | 18031 | 48331 | -| Zcash Testnet | 19132 | 19032 | 18032 | 48332 | -| Dash Testnet | 19133 | 19033 | 18033 | 48333 | -| Litecoin Testnet | 19134 | 19034 | 18034 | 48334 | -| Bitcoin Gold Testnet | 19135 | 19035 | 18035 | 48335 | -| Ethereum Testnet Holesky Archive | 19136 | 19036 | 18036 | 18136 http, 18536 authrpc, 48336 p2p | -| Dogecoin Testnet | 19138 | 19038 | 18038 | 48338 | -| Vertcoin Testnet | 19140 | 19040 | 18040 | 48340 | -| Monacoin Testnet | 19141 | 19041 | 18041 | 48341 | -| DigiByte Testnet | 19142 | 19042 | 18042 | 48342 | -| Groestlcoin Testnet | 19145 | 19045 | 18045 | 48345 | -| Groestlcoin Regtest | 19146 | 19046 | 18046 | 48346 | -| Groestlcoin Signet | 19147 | 19047 | 18047 | 48347 | -| PIVX Testnet | 19149 | 19049 | 18049 | 48349 | -| Koto Testnet | 19151 | 19051 | 18051 | 48351 | -| Decred Testnet | 19161 | 19061 | 18061 | 48361 | -| Flo Testnet | 19166 | 19066 | 18066 | 48366 | -| Ethereum Testnet Sepolia | 19176 | 19076 | 18076 | 18176 http, 18576 authrpc, 48376 p2p | -| Ethereum Testnet Sepolia Archive | 19186 | 19086 | 18086 | 18186 http, 18586 authrpc, 48386 p2p | -| Qtum Testnet | 19188 | 19088 | 18088 | 48388 | -| Omotenashicoin Testnet | 19189 | 19089 | 18089 | 48389 | +| coin | blockbook public | blockbook internal | backend rpc | backend service ports (zmq) | +|----------------------------------|------------------|--------------------|-------------|-----------------------------------------------------| +| Ethereum Archive | 9116 | 9016 | 8016 | 38316 p2p, 8116 http, 8516 authrpc | +| Bitcoin | 9130 | 9030 | 8030 | 38330 | +| Bitcoin Cash | 9131 | 9031 | 8031 | 38331 | +| Zcash | 9132 | 9032 | 8032 | 38332 | +| Dash | 9133 | 9033 | 8033 | 38333 | +| Litecoin | 9134 | 9034 | 8034 | 38334 | +| Bitcoin Gold | 9135 | 9035 | 8035 | 38335 | +| Ethereum | 9136 | 9036 | 8036 | 38336 p2p, 8136 http, 8536 authrpc | +| Ethereum Classic | 9137 | 9037 | 8037 | 38337 p2p, 8137 http | +| Dogecoin | 9138 | 9038 | 8038 | 38338 | +| Namecoin | 9139 | 9039 | 8039 | 38339 | +| Vertcoin | 9140 | 9040 | 8040 | 38340 | +| Monacoin | 9141 | 9041 | 8041 | 38341 | +| DigiByte | 9142 | 9042 | 8042 | 38342 | +| Myriad | 9143 | 9043 | 8043 | 38343 | +| GameCredits | 9144 | 9044 | 8044 | 38344 | +| Groestlcoin | 9145 | 9045 | 8045 | 38345 | +| Bitcoin Cash SV | 9146 | 9046 | 8046 | 38346 | +| Liquid | 9147 | 9047 | 8047 | 38347 | +| Fujicoin | 9148 | 9048 | 8048 | 38348 | +| PIVX | 9149 | 9049 | 8049 | 38349 | +| Firo | 9150 | 9050 | 8050 | 38350 | +| Koto | 9151 | 9051 | 8051 | 38351 | +| Bellcoin | 9152 | 9052 | 8052 | 38352 | +| NULS | 9153 | 9053 | 8053 | 38353 | +| Bitcore | 9154 | 9054 | 8054 | 38354 | +| Viacoin | 9155 | 9055 | 8055 | 38355 | +| VIPSTARCOIN | 9156 | 9056 | 8056 | 38356 | +| MonetaryUnit | 9157 | 9057 | 8057 | 38357 | +| Flux | 9158 | 9058 | 8058 | 38358 | +| Ravencoin | 9159 | 9059 | 8059 | 38359 | +| Ritocoin | 9160 | 9060 | 8060 | 38360 | +| Decred | 9161 | 9061 | 8061 | 38361 | +| SnowGem | 9162 | 9062 | 8062 | 38362 | +| BNB Smart Chain | 9164 | 9064 | 8064 | 38364 p2p, 8164 http | +| BNB Smart Chain Archive | 9165 | 9065 | 8065 | 38365 p2p, 8165 http | +| Flo | 9166 | 9066 | 8066 | 38366 | +| Polis | 9167 | 9067 | 8067 | 38367 | +| Polygon | 9170 | 9070 | 8070 | 38370 p2p, 8170 http | +| Polygon Archive | 9172 | 9072 | 8072 | 38372 p2p, 8172 http | +| Gnosis | 9174 | 9074 | 8074 | 38374 p2p, 8174 http, 8574 authrpc | +| Gnosis Archive | 9176 | 9076 | 8076 | 38376 p2p, 8176 http, 8576 authrpc | +| Qtum | 9188 | 9088 | 8088 | 38388 | +| Divi Project | 9189 | 9089 | 8089 | 38389 | +| CPUchain | 9190 | 9090 | 8090 | 38390 | +| DeepOnion | 9191 | 9091 | 8091 | 38391 | +| Unobtanium | 9192 | 9092 | 65535 | 38392 | +| Omotenashicoin | 9194 | 9094 | 8094 | 38394 | +| BitZeny | 9195 | 9095 | 8095 | 38395 | +| Trezarcoin | 9196 | 9096 | 8096 | 38396 | +| eCash | 9197 | 9097 | 8097 | 38397 | +| Avalanche | 9198 | 9098 | 8098 | 38398 p2p | +| Avalanche Archive | 9199 | 9099 | 8099 | 38399 p2p | +| Ethereum Testnet Holesky | 19116 | 19016 | 18016 | 18116 http, 18516 authrpc, 48316 p2p | +| Bitcoin Signet | 19120 | 19020 | 18020 | 48320 | +| Bitcoin Regtest | 19121 | 19021 | 18021 | 48321 | +| Bitcoin Testnet | 19130 | 19030 | 18030 | 48330 | +| Bitcoin Cash Testnet | 19131 | 19031 | 18031 | 48331 | +| Zcash Testnet | 19132 | 19032 | 18032 | 48332 | +| Dash Testnet | 19133 | 19033 | 18033 | 48333 | +| Litecoin Testnet | 19134 | 19034 | 18034 | 48334 | +| Bitcoin Gold Testnet | 19135 | 19035 | 18035 | 48335 | +| Ethereum Testnet Holesky Archive | 19136 | 19036 | 18036 | 18136 http, 18136 torrent, 18536 authrpc, 48336 p2p | +| Dogecoin Testnet | 19138 | 19038 | 18038 | 48338 | +| Vertcoin Testnet | 19140 | 19040 | 18040 | 48340 | +| Monacoin Testnet | 19141 | 19041 | 18041 | 48341 | +| DigiByte Testnet | 19142 | 19042 | 18042 | 48342 | +| Groestlcoin Testnet | 19145 | 19045 | 18045 | 48345 | +| Groestlcoin Regtest | 19146 | 19046 | 18046 | 48346 | +| Groestlcoin Signet | 19147 | 19047 | 18047 | 48347 | +| PIVX Testnet | 19149 | 19049 | 18049 | 48349 | +| Koto Testnet | 19151 | 19051 | 18051 | 48351 | +| Decred Testnet | 19161 | 19061 | 18061 | 48361 | +| Flo Testnet | 19166 | 19066 | 18066 | 48366 | +| Ethereum Testnet Sepolia | 19176 | 19076 | 18076 | 18176 http, 18576 authrpc, 48376 p2p | +| Ethereum Testnet Sepolia Archive | 19186 | 19086 | 18086 | 18186 http, 18186 torrent, 18586 authrpc, 48386 p2p | +| Qtum Testnet | 19188 | 19088 | 18088 | 48388 | +| Omotenashicoin Testnet | 19189 | 19089 | 18089 | 48389 | > NOTE: This document is generated from coin definitions in `configs/coins` using command `go run contrib/scripts/check-and-generate-port-registry.go -w`.