From b0859432fb5f6f876944f713c6870ece6cd48bb6 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 15 Mar 2022 12:32:05 +0100 Subject: [PATCH 01/29] Add command to download a contract from a remote network into our local chain of choice --- .../Commands/ContractCommand.Download.cs | 75 ++++++++++++++++ src/neoxp/Commands/ContractCommand.cs | 2 +- src/neoxp/IExpressNode.cs | 4 + src/neoxp/Node/Modules/ExpressRpcMethods.cs | 86 +++++++++++++++++++ src/neoxp/Node/OfflineNode.cs | 26 ++++++ src/neoxp/Node/OnlineNode.cs | 9 ++ 6 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 src/neoxp/Commands/ContractCommand.Download.cs diff --git a/src/neoxp/Commands/ContractCommand.Download.cs b/src/neoxp/Commands/ContractCommand.Download.cs new file mode 100644 index 00000000..2922337d --- /dev/null +++ b/src/neoxp/Commands/ContractCommand.Download.cs @@ -0,0 +1,75 @@ +using System; +using System.ComponentModel.DataAnnotations; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using McMaster.Extensions.CommandLineUtils; +using Neo; +using Neo.IO.Json; +using Neo.Network.RPC; + +namespace NeoExpress.Commands +{ + partial class ContractCommand + { + [Command(Name = "download", Description = "Download contract with storage from remote chain into local chain")] + internal class Download + { + readonly ExpressChainManagerFactory chainManagerFactory; + private RpcClient rpcClient = null!; + + public Download(ExpressChainManagerFactory chainManagerFactory) + { + this.chainManagerFactory = chainManagerFactory; + } + + [Argument(0, Description = "Contract invocation hash")] + [Required] + internal string Contract { get; init; } = string.Empty; + + [Argument(1, Description = "Source network RPC address")] + internal string Source { get; init; } = string.Empty; + + [Option(Description = "Path to neo-express data file")] + internal string Input { get; init; } = string.Empty; + + internal async Task ExecuteAsync(TextWriter writer) + { + var (chainManager, _) = chainManagerFactory.LoadChain(Input); + var expressNode = chainManager.GetExpressNode(); + + this.rpcClient = new RpcClient(new Uri(Source)); + + if (!UInt160.TryParse(Contract, out _)) + { + await writer.WriteLineAsync($"Invalid contract hash: {Contract} ").ConfigureAwait(false); + } + else + { + // 1. Get ContractState + var state = await this.rpcClient.GetContractStateAsync(Contract).ConfigureAwait(false); + + // 2. Get Full storage of the contract + var storage_pairs = (JArray)await this.rpcClient.RpcSendAsync("getfullstorage", Contract).ConfigureAwait(false); + + await writer.WriteLineAsync(storage_pairs.ToString(true)).ConfigureAwait(false); + await expressNode.PersistContractAsync(state, storage_pairs); + } + } + + internal async Task OnExecuteAsync(CommandLineApplication app, IConsole console) + { + try + { + await ExecuteAsync(console.Out).ConfigureAwait(false); + return 0; + } + catch (Exception ex) + { + app.WriteException(ex); + return 1; + } + } + } + } +} \ No newline at end of file diff --git a/src/neoxp/Commands/ContractCommand.cs b/src/neoxp/Commands/ContractCommand.cs index c2fd9c66..68c238f3 100644 --- a/src/neoxp/Commands/ContractCommand.cs +++ b/src/neoxp/Commands/ContractCommand.cs @@ -3,7 +3,7 @@ namespace NeoExpress.Commands { [Command("contract", Description = "Manage smart contracts")] - [Subcommand(typeof(Deploy), typeof(Get), typeof(Hash), typeof(Invoke), typeof(List), typeof(Run), typeof(Storage))] + [Subcommand(typeof(Deploy), typeof(Get), typeof(Hash), typeof(Invoke), typeof(List), typeof(Run), typeof(Storage), typeof(Download))] partial class ContractCommand { internal int OnExecute(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/IExpressNode.cs b/src/neoxp/IExpressNode.cs index b75c8dbc..52530d32 100644 --- a/src/neoxp/IExpressNode.cs +++ b/src/neoxp/IExpressNode.cs @@ -4,8 +4,10 @@ using System.Threading.Tasks; using Neo; using Neo.Cryptography.ECC; +using Neo.IO.Json; using Neo.Network.P2P.Payloads; using Neo.Network.RPC.Models; +using Neo.SmartContract; using Neo.SmartContract.Manifest; using Neo.SmartContract.Native; using Neo.VM; @@ -39,5 +41,7 @@ enum CheckpointMode { Online, Offline } Task> ListOracleRequestsAsync(); Task> ListStoragesAsync(UInt160 scriptHash); Task> ListTokenContractsAsync(); + + Task PersistContractAsync(ContractState state, JArray storagePairs); } } diff --git a/src/neoxp/Node/Modules/ExpressRpcMethods.cs b/src/neoxp/Node/Modules/ExpressRpcMethods.cs index e10ac93c..6cbf0a54 100644 --- a/src/neoxp/Node/Modules/ExpressRpcMethods.cs +++ b/src/neoxp/Node/Modules/ExpressRpcMethods.cs @@ -8,8 +8,10 @@ using Neo.IO; using Neo.IO.Json; using Neo.Network.P2P.Payloads; +using Neo.Persistence; using Neo.Plugins; using Neo.SmartContract; +using Neo.SmartContract.Manifest; using Neo.SmartContract.Native; using Neo.VM; using Neo.Wallets; @@ -403,6 +405,90 @@ public JObject GetNep11Properties(JArray @params) } return json; } + + private const byte Prefix_Contract = 8; + private const byte Prefix_NextAvailableId = 15; + + private int LastUsedContractId(DataCache snapshot) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.TryGet(key); + return (int)(BigInteger)item; + } + + private void SetLastUsedContractId(DataCache snapshot, int newId) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.GetAndChange(key); + item.Set(newId); + } + + private int GetNextAvailableId(DataCache snapshot) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.GetAndChange(key); + int value = (int)(BigInteger)item; + item.Add(1); + return value; + } + + [RpcMethod] + public JObject? ExpressPersistContract(JObject @params) + { + var state = @params[0]["state"]; + var storagePairs = (JArray) @params[0]["storage"]; + var snapshot = neoSystem.GetSnapshot(); + + var nef = new NefFile + { + CheckSum = (uint)state["nef"]["checksum"].AsNumber(), + Compiler = state["nef"]["compiler"].AsString(), + Source = state["nef"]["source"].AsString(), + Tokens = new MethodToken[0], // TODO: support parsing MethodTokens or use fix of https://github.com/neo-project/neo/issues/2674 + Script = Convert.FromBase64String(state["nef"]["script"].AsString()) + }; + + ContractManifest m = ContractManifest.FromJson(state["manifest"]); + + // Our local chain might already be using the contract id of the pulled contract, we need to check for this + // to avoid having contracts with duplicate id's. This is important because the contract id is part of the + // StorageContext used with Storage syscalls and else we'll potentially override storage keys or iterate + // over keys that shouldn't exist for one of the contracts. + int contractId = (int)state["id"].AsNumber(); + if (contractId <= LastUsedContractId(snapshot)) + { + contractId = GetNextAvailableId(snapshot); + } + else + { + // Update available id such that a regular contract deploy will use the right next id; + SetLastUsedContractId(snapshot, contractId); + } + + var c = new ContractState + { + Id = contractId, + Hash = UInt160.Parse(state["hash"].AsString()), + UpdateCounter = (ushort)state["updatecounter"].AsNumber(), + Nef = nef, + Manifest = m + }; + + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(c.Hash); + snapshot.Add(key, new StorageItem(c)); + + foreach (var pair in storagePairs) + { + snapshot.Add( + new StorageKey { Id = c.Id, Key = Convert.FromBase64String(pair["k"].AsString())}, + new StorageItem(Convert.FromBase64String(pair["v"].AsString())) + ); + } + + snapshot.Commit(); + + return true; + } static readonly IReadOnlySet nep11PropertyNames = new HashSet { diff --git a/src/neoxp/Node/OfflineNode.cs b/src/neoxp/Node/OfflineNode.cs index 4fced3a7..5ad0830a 100644 --- a/src/neoxp/Node/OfflineNode.cs +++ b/src/neoxp/Node/OfflineNode.cs @@ -11,6 +11,7 @@ using Neo.Cryptography; using Neo.Cryptography.ECC; using Neo.IO; +using Neo.IO.Json; using Neo.Ledger; using Neo.Network.P2P.Payloads; using Neo.Network.RPC; @@ -406,5 +407,30 @@ IReadOnlyList ListStorages(UInt160 scriptHash) public Task> ListStoragesAsync(UInt160 scriptHash) => MakeAsync(() => ListStorages(scriptHash)); + + private const byte Prefix_Contract = 8; + + bool PersistContract(ContractState state, JArray storagePairs) + { + var snapshot = neoSystem.StoreView.CreateSnapshot(); + + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(state.Hash); + snapshot.Add(key, new StorageItem(state)); + + foreach (var pair in storagePairs) + { + snapshot.Add( + new StorageKey { Id = state.Id, Key = Convert.FromBase64String(pair["k"].AsString())}, + new StorageItem(Convert.FromBase64String(pair["v"].AsString())) + ); + } + + snapshot.Commit(); + return true; + } + + public Task PersistContractAsync(ContractState state, JArray storagePairs) + => MakeAsync(() => PersistContract(state, storagePairs)); + } } diff --git a/src/neoxp/Node/OnlineNode.cs b/src/neoxp/Node/OnlineNode.cs index c44381af..954d472d 100644 --- a/src/neoxp/Node/OnlineNode.cs +++ b/src/neoxp/Node/OnlineNode.cs @@ -328,5 +328,14 @@ public async Task> ListStoragesAsync(UInt160 scrip return Array.Empty(); } + + public async Task PersistContractAsync(ContractState state, JArray storagePairs) + { + JObject o = new JObject(); + o["state"] = state.ToJson(); + o["storage"] = storagePairs; + var json = await rpcClient.RpcSendAsync("expresspersistcontract", o).ConfigureAwait(false); + return true; + } } } From f389c64a704d3d223702e87e28666ccae0949e20 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Fri, 18 Mar 2022 12:18:11 +0100 Subject: [PATCH 02/29] Process feedback --- .../Commands/ContractCommand.Download.cs | 39 ++++---- src/neoxp/Commands/ContractCommand.cs | 2 +- src/neoxp/IExpressNode.cs | 3 +- src/neoxp/Node/ExpressOracle.cs | 57 ++++++++++++ src/neoxp/Node/Modules/ExpressRpcMethods.cs | 88 +++---------------- src/neoxp/Node/OfflineNode.cs | 24 +---- src/neoxp/Node/OnlineNode.cs | 18 +++- 7 files changed, 110 insertions(+), 121 deletions(-) diff --git a/src/neoxp/Commands/ContractCommand.Download.cs b/src/neoxp/Commands/ContractCommand.Download.cs index 2922337d..1b30b667 100644 --- a/src/neoxp/Commands/ContractCommand.Download.cs +++ b/src/neoxp/Commands/ContractCommand.Download.cs @@ -16,7 +16,6 @@ partial class ContractCommand internal class Download { readonly ExpressChainManagerFactory chainManagerFactory; - private RpcClient rpcClient = null!; public Download(ExpressChainManagerFactory chainManagerFactory) { @@ -27,8 +26,8 @@ public Download(ExpressChainManagerFactory chainManagerFactory) [Required] internal string Contract { get; init; } = string.Empty; - [Argument(1, Description = "Source network RPC address")] - internal string Source { get; init; } = string.Empty; + [Argument(1, Description = "URL of Neo JSON-RPC Node\nSpecify MainNet (default), TestNet or JSON-RPC URL")] + internal string RpcUri { get; } = string.Empty; [Option(Description = "Path to neo-express data file")] internal string Input { get; init; } = string.Empty; @@ -38,23 +37,25 @@ internal async Task ExecuteAsync(TextWriter writer) var (chainManager, _) = chainManagerFactory.LoadChain(Input); var expressNode = chainManager.GetExpressNode(); - this.rpcClient = new RpcClient(new Uri(Source)); + if (!UInt160.TryParse(Contract, out var contractHash)) + throw new ArgumentException($"Invalid contract hash: \"{Contract}\""); + + if (!TransactionExecutor.TryParseRpcUri(RpcUri, out var uri)) + throw new ArgumentException($"Invalid RpcUri value \"{RpcUri}\""); + + using var rpcClient = new RpcClient(uri); + var stateAPI = new StateAPI(rpcClient); + + var stateHeight = await stateAPI.GetStateHeightAsync(); + if (stateHeight.localRootIndex is null) + throw new Exception("Null \"localRootIndex\" in state height response"); + var stateRoot = await stateAPI.GetStateRootAsync(stateHeight.localRootIndex.Value); + var states = await stateAPI.FindStatesAsync(stateRoot.RootHash, contractHash, new byte[0]); + + var contractState = await rpcClient.GetContractStateAsync(Contract).ConfigureAwait(false); + + await expressNode.PersistContractAsync(contractState, states.Results); - if (!UInt160.TryParse(Contract, out _)) - { - await writer.WriteLineAsync($"Invalid contract hash: {Contract} ").ConfigureAwait(false); - } - else - { - // 1. Get ContractState - var state = await this.rpcClient.GetContractStateAsync(Contract).ConfigureAwait(false); - - // 2. Get Full storage of the contract - var storage_pairs = (JArray)await this.rpcClient.RpcSendAsync("getfullstorage", Contract).ConfigureAwait(false); - - await writer.WriteLineAsync(storage_pairs.ToString(true)).ConfigureAwait(false); - await expressNode.PersistContractAsync(state, storage_pairs); - } } internal async Task OnExecuteAsync(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/Commands/ContractCommand.cs b/src/neoxp/Commands/ContractCommand.cs index 68c238f3..4a625272 100644 --- a/src/neoxp/Commands/ContractCommand.cs +++ b/src/neoxp/Commands/ContractCommand.cs @@ -3,7 +3,7 @@ namespace NeoExpress.Commands { [Command("contract", Description = "Manage smart contracts")] - [Subcommand(typeof(Deploy), typeof(Get), typeof(Hash), typeof(Invoke), typeof(List), typeof(Run), typeof(Storage), typeof(Download))] + [Subcommand(typeof(Deploy), typeof(Download), typeof(Get), typeof(Hash), typeof(Invoke), typeof(List), typeof(Run), typeof(Storage))] partial class ContractCommand { internal int OnExecute(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/IExpressNode.cs b/src/neoxp/IExpressNode.cs index 52530d32..e182ca56 100644 --- a/src/neoxp/IExpressNode.cs +++ b/src/neoxp/IExpressNode.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using Neo; using Neo.Cryptography.ECC; -using Neo.IO.Json; using Neo.Network.P2P.Payloads; using Neo.Network.RPC.Models; using Neo.SmartContract; @@ -42,6 +41,6 @@ enum CheckpointMode { Online, Offline } Task> ListStoragesAsync(UInt160 scriptHash); Task> ListTokenContractsAsync(); - Task PersistContractAsync(ContractState state, JArray storagePairs); + Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs); } } diff --git a/src/neoxp/Node/ExpressOracle.cs b/src/neoxp/Node/ExpressOracle.cs index 3031dd55..09d11be6 100644 --- a/src/neoxp/Node/ExpressOracle.cs +++ b/src/neoxp/Node/ExpressOracle.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Numerics; using Neo; using Neo.BlockchainToolkit.Models; using Neo.Cryptography.ECC; @@ -138,5 +139,61 @@ public static void SignOracleResponseTransaction(ProtocolSettings settings, Expr return tx; } + + public const byte Prefix_Contract = 8; + private const byte Prefix_NextAvailableId = 15; + + private static int LastUsedContractId(DataCache snapshot) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.TryGet(key); + return (int)(BigInteger)item; + } + + private static void SetLastUsedContractId(DataCache snapshot, int newId) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.GetAndChange(key); + item.Set(newId); + } + + private static int GetNextAvailableId(DataCache snapshot) + { + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); + StorageItem item = snapshot.GetAndChange(key); + int value = (int)(BigInteger)item; + item.Add(1); + return value; + } + + public static bool PersistContract(SnapshotCache snapshot, ContractState state, (byte[] key, byte[] value)[] storagePairs) + { + // Our local chain might already be using the contract id of the pulled contract, we need to check for this + // to avoid having contracts with duplicate id's. This is important because the contract id is part of the + // StorageContext used with Storage syscalls and else we'll potentially override storage keys or iterate + // over keys that shouldn't exist for one of the contracts. + if (state.Id <= LastUsedContractId(snapshot)) + { + state.Id = GetNextAvailableId(snapshot); + } + else + { + // Update available id such that a regular contract deploy will use the right next id; + SetLastUsedContractId(snapshot, state.Id); + } + + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(state.Hash); + snapshot.Add(key, new StorageItem(state)); + + foreach (var pair in storagePairs) + { + snapshot.Add( + new StorageKey { Id = state.Id, Key = pair.key}, + new StorageItem(pair.value) + ); + } + snapshot.Commit(); + return true; + } } } diff --git a/src/neoxp/Node/Modules/ExpressRpcMethods.cs b/src/neoxp/Node/Modules/ExpressRpcMethods.cs index 6cbf0a54..b40a818a 100644 --- a/src/neoxp/Node/Modules/ExpressRpcMethods.cs +++ b/src/neoxp/Node/Modules/ExpressRpcMethods.cs @@ -3,11 +3,13 @@ using System.Linq; using System.Numerics; using System.Threading; +using Microsoft.EntityFrameworkCore.Metadata.Internal; using Neo; using Neo.BlockchainToolkit.Persistence; using Neo.IO; using Neo.IO.Json; using Neo.Network.P2P.Payloads; +using Neo.Network.RPC; using Neo.Persistence; using Neo.Plugins; using Neo.SmartContract; @@ -17,6 +19,8 @@ using Neo.Wallets; using NeoExpress.Models; using ByteString = Neo.VM.Types.ByteString; +using RpcException = Neo.Plugins.RpcException; +using Utility = Neo.Utility; namespace NeoExpress.Node { @@ -406,88 +410,22 @@ public JObject GetNep11Properties(JArray @params) return json; } - private const byte Prefix_Contract = 8; - private const byte Prefix_NextAvailableId = 15; - - private int LastUsedContractId(DataCache snapshot) - { - StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); - StorageItem item = snapshot.TryGet(key); - return (int)(BigInteger)item; - } - - private void SetLastUsedContractId(DataCache snapshot, int newId) - { - StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); - StorageItem item = snapshot.GetAndChange(key); - item.Set(newId); - } - - private int GetNextAvailableId(DataCache snapshot) - { - StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_NextAvailableId); - StorageItem item = snapshot.GetAndChange(key); - int value = (int)(BigInteger)item; - item.Add(1); - return value; - } - [RpcMethod] public JObject? ExpressPersistContract(JObject @params) { - var state = @params[0]["state"]; - var storagePairs = (JArray) @params[0]["storage"]; - var snapshot = neoSystem.GetSnapshot(); - - var nef = new NefFile - { - CheckSum = (uint)state["nef"]["checksum"].AsNumber(), - Compiler = state["nef"]["compiler"].AsString(), - Source = state["nef"]["source"].AsString(), - Tokens = new MethodToken[0], // TODO: support parsing MethodTokens or use fix of https://github.com/neo-project/neo/issues/2674 - Script = Convert.FromBase64String(state["nef"]["script"].AsString()) - }; - ContractManifest m = ContractManifest.FromJson(state["manifest"]); - - // Our local chain might already be using the contract id of the pulled contract, we need to check for this - // to avoid having contracts with duplicate id's. This is important because the contract id is part of the - // StorageContext used with Storage syscalls and else we'll potentially override storage keys or iterate - // over keys that shouldn't exist for one of the contracts. - int contractId = (int)state["id"].AsNumber(); - if (contractId <= LastUsedContractId(snapshot)) - { - contractId = GetNextAvailableId(snapshot); - } - else + var state = RpcClient.ContractStateFromJson(@params[0]["state"]); + var storagePairs = new (byte[] Key, byte[] value)[0]; + + foreach (var pair in (JArray)@params[0]["storage"]) { - // Update available id such that a regular contract deploy will use the right next id; - SetLastUsedContractId(snapshot, contractId); + storagePairs.Append(( + Convert.FromBase64String(pair["key"].AsString()), + Convert.FromBase64String(pair["value"].AsString()) + )); } - var c = new ContractState - { - Id = contractId, - Hash = UInt160.Parse(state["hash"].AsString()), - UpdateCounter = (ushort)state["updatecounter"].AsNumber(), - Nef = nef, - Manifest = m - }; - - StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(c.Hash); - snapshot.Add(key, new StorageItem(c)); - - foreach (var pair in storagePairs) - { - snapshot.Add( - new StorageKey { Id = c.Id, Key = Convert.FromBase64String(pair["k"].AsString())}, - new StorageItem(Convert.FromBase64String(pair["v"].AsString())) - ); - } - - snapshot.Commit(); - - return true; + return ExpressOracle.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); } static readonly IReadOnlySet nep11PropertyNames = new HashSet diff --git a/src/neoxp/Node/OfflineNode.cs b/src/neoxp/Node/OfflineNode.cs index 5ad0830a..ea92312b 100644 --- a/src/neoxp/Node/OfflineNode.cs +++ b/src/neoxp/Node/OfflineNode.cs @@ -104,7 +104,7 @@ Task MakeAsync(Func func) return Task.FromException(ex); } } - + IExpressNode.CheckpointMode CreateCheckpoint(string checkPointPath) { var multiSigAccount = nodeWallet.GetMultiSigAccounts().Single(); @@ -408,28 +408,12 @@ IReadOnlyList ListStorages(UInt160 scriptHash) public Task> ListStoragesAsync(UInt160 scriptHash) => MakeAsync(() => ListStorages(scriptHash)); - private const byte Prefix_Contract = 8; - - bool PersistContract(ContractState state, JArray storagePairs) + bool PersistContract(ContractState state, (byte[] key, byte[] value)[] storagePairs) { - var snapshot = neoSystem.StoreView.CreateSnapshot(); - - StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(state.Hash); - snapshot.Add(key, new StorageItem(state)); - - foreach (var pair in storagePairs) - { - snapshot.Add( - new StorageKey { Id = state.Id, Key = Convert.FromBase64String(pair["k"].AsString())}, - new StorageItem(Convert.FromBase64String(pair["v"].AsString())) - ); - } - - snapshot.Commit(); - return true; + return ExpressOracle.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); } - public Task PersistContractAsync(ContractState state, JArray storagePairs) + public Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs) => MakeAsync(() => PersistContract(state, storagePairs)); } diff --git a/src/neoxp/Node/OnlineNode.cs b/src/neoxp/Node/OnlineNode.cs index 954d472d..4ec6df28 100644 --- a/src/neoxp/Node/OnlineNode.cs +++ b/src/neoxp/Node/OnlineNode.cs @@ -329,13 +329,23 @@ public async Task> ListStoragesAsync(UInt160 scrip return Array.Empty(); } - public async Task PersistContractAsync(ContractState state, JArray storagePairs) + public async Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs) { JObject o = new JObject(); o["state"] = state.ToJson(); - o["storage"] = storagePairs; - var json = await rpcClient.RpcSendAsync("expresspersistcontract", o).ConfigureAwait(false); - return true; + + JArray storage = new JArray(); + foreach (var pair in storagePairs) + { + JObject kv = new JObject(); + kv["key"] = Convert.ToBase64String(pair.key); + kv["value"] = Convert.ToBase64String(pair.value); + storage.Add(kv); + } + o["storage"] = storage; + + var response = await rpcClient.RpcSendAsync("expresspersistcontract", o).ConfigureAwait(false); + return response.AsBoolean(); } } } From 4fd6638b61dbde864eb976b6c9c024d1efa836e7 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Mon, 21 Mar 2022 14:26:57 +0100 Subject: [PATCH 03/29] Add ExpressFindStatesAsync without key decoding and with paging support Add {} around exceptions Rename ExpressOracle --- .../Commands/ContractCommand.Download.cs | 13 ++++-- src/neoxp/Extensions/ExpressRpcFoundStates.cs | 34 ++++++++++++++ src/neoxp/Extensions/Extensions.cs | 44 +++++++++++++++++++ src/neoxp/IExpressNode.cs | 2 +- src/neoxp/Node/Modules/ExpressRpcMethods.cs | 22 +++------- .../Node/{ExpressOracle.cs => NodeUtility.cs} | 10 ++--- src/neoxp/Node/OfflineNode.cs | 10 ++--- src/neoxp/Node/OnlineNode.cs | 12 ++--- 8 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 src/neoxp/Extensions/ExpressRpcFoundStates.cs rename src/neoxp/Node/{ExpressOracle.cs => NodeUtility.cs} (96%) diff --git a/src/neoxp/Commands/ContractCommand.Download.cs b/src/neoxp/Commands/ContractCommand.Download.cs index 1b30b667..c9b8aa50 100644 --- a/src/neoxp/Commands/ContractCommand.Download.cs +++ b/src/neoxp/Commands/ContractCommand.Download.cs @@ -38,24 +38,29 @@ internal async Task ExecuteAsync(TextWriter writer) var expressNode = chainManager.GetExpressNode(); if (!UInt160.TryParse(Contract, out var contractHash)) - throw new ArgumentException($"Invalid contract hash: \"{Contract}\""); + { + throw new ArgumentException($"Invalid contract hash: \"{Contract}\""); + } if (!TransactionExecutor.TryParseRpcUri(RpcUri, out var uri)) + { throw new ArgumentException($"Invalid RpcUri value \"{RpcUri}\""); + } using var rpcClient = new RpcClient(uri); var stateAPI = new StateAPI(rpcClient); var stateHeight = await stateAPI.GetStateHeightAsync(); if (stateHeight.localRootIndex is null) + { throw new Exception("Null \"localRootIndex\" in state height response"); - var stateRoot = await stateAPI.GetStateRootAsync(stateHeight.localRootIndex.Value); - var states = await stateAPI.FindStatesAsync(stateRoot.RootHash, contractHash, new byte[0]); + } + var stateRoot = await stateAPI.GetStateRootAsync(stateHeight.localRootIndex.Value); + var states = await rpcClient.ExpressFindStatesAsync(stateRoot.RootHash, contractHash, new byte[0]); var contractState = await rpcClient.GetContractStateAsync(Contract).ConfigureAwait(false); await expressNode.PersistContractAsync(contractState, states.Results); - } internal async Task OnExecuteAsync(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/Extensions/ExpressRpcFoundStates.cs b/src/neoxp/Extensions/ExpressRpcFoundStates.cs new file mode 100644 index 00000000..e15fd4ff --- /dev/null +++ b/src/neoxp/Extensions/ExpressRpcFoundStates.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Neo.IO.Json; + +namespace NeoExpress +{ + public class ExpressRpcFoundStates + { + public bool Truncated; + public (string key, string value)[] Results; + public byte[]? FirstProof; + public byte[]? LastProof; + + public static ExpressRpcFoundStates FromJson(JObject json) + { + return new ExpressRpcFoundStates + { + Truncated = json["truncated"].AsBoolean(), + Results = ((JArray)json["results"]) + .Select(j => ( + j["key"].AsString(), + j["value"].AsString() + )) + .ToArray(), + FirstProof = ProofFromJson(json["firstProof"]), + LastProof = ProofFromJson(json["lastProof"]), + }; + } + + static byte[] ProofFromJson(JObject json) + => json == null ? null : Convert.FromBase64String(json.AsString()); + } +} \ No newline at end of file diff --git a/src/neoxp/Extensions/Extensions.cs b/src/neoxp/Extensions/Extensions.cs index e1cb17c5..9bfb5b6f 100644 --- a/src/neoxp/Extensions/Extensions.cs +++ b/src/neoxp/Extensions/Extensions.cs @@ -7,6 +7,7 @@ using McMaster.Extensions.CommandLineUtils; using Neo; using Neo.BlockchainToolkit; +using Neo.IO.Json; using Neo.Network.P2P.Payloads; using Neo.Network.RPC; using Neo.Network.RPC.Models; @@ -261,5 +262,48 @@ static bool TryGetNEP6Wallet(string path, string password, ProtocolSettings sett } } } + + + private static JObject[] MakeFindStatesParams( + UInt256 rootHash, + UInt160 scriptHash, + ReadOnlySpan prefix, + ReadOnlySpan from = default (ReadOnlySpan), + int? count = null) + { + JObject[] jobjectArray = new JObject[count.HasValue ? 5 : 4]; + jobjectArray[0] = (JObject) rootHash.ToString(); + jobjectArray[1] = (JObject) scriptHash.ToString(); + jobjectArray[2] = (JObject) Convert.ToBase64String(prefix); + jobjectArray[3] = (JObject) Convert.ToBase64String(from); + if (count.HasValue) + jobjectArray[4] = (JObject) (double) count.Value; + return jobjectArray; + } + public static async Task ExpressFindStatesAsync(this RpcClient rpcClient, UInt256 rootHash, UInt160 contractScriptHash, ReadOnlyMemory prefix, ReadOnlyMemory from = default, int? pageSize = null) + { + ExpressRpcFoundStates? states = null; + var start = from.ToArray(); + while (true) + { + var @params = MakeFindStatesParams(rootHash, contractScriptHash, prefix.Span, start, pageSize); + var response = await rpcClient.RpcSendAsync("findstates", @params).ConfigureAwait(false); + var foundStates = ExpressRpcFoundStates.FromJson(response); + + if (states is null) + { + states = foundStates; + } else + { + states.Results = states.Results.Concat(foundStates.Results).ToArray(); + } + + if (!foundStates.Truncated || foundStates.Results.Length == 0) + { + return states; + } + start = Convert.FromBase64String(foundStates.Results[foundStates.Results.Length - 1].key); + } + } } } diff --git a/src/neoxp/IExpressNode.cs b/src/neoxp/IExpressNode.cs index e182ca56..063d75bb 100644 --- a/src/neoxp/IExpressNode.cs +++ b/src/neoxp/IExpressNode.cs @@ -41,6 +41,6 @@ enum CheckpointMode { Online, Offline } Task> ListStoragesAsync(UInt160 scriptHash); Task> ListTokenContractsAsync(); - Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs); + Task PersistContractAsync(ContractState state, (string key, string value)[] storagePairs); } } diff --git a/src/neoxp/Node/Modules/ExpressRpcMethods.cs b/src/neoxp/Node/Modules/ExpressRpcMethods.cs index b40a818a..48c17113 100644 --- a/src/neoxp/Node/Modules/ExpressRpcMethods.cs +++ b/src/neoxp/Node/Modules/ExpressRpcMethods.cs @@ -3,17 +3,14 @@ using System.Linq; using System.Numerics; using System.Threading; -using Microsoft.EntityFrameworkCore.Metadata.Internal; using Neo; using Neo.BlockchainToolkit.Persistence; using Neo.IO; using Neo.IO.Json; using Neo.Network.P2P.Payloads; using Neo.Network.RPC; -using Neo.Persistence; using Neo.Plugins; using Neo.SmartContract; -using Neo.SmartContract.Manifest; using Neo.SmartContract.Native; using Neo.VM; using Neo.Wallets; @@ -239,7 +236,7 @@ public JObject ExpressListTokenContracts(JArray _) var height = NativeContract.Ledger.CurrentIndex(snapshot) + 1; var oracleNodes = NativeContract.RoleManagement.GetDesignatedByRole(snapshot, Role.Oracle, height); var request = NativeContract.Oracle.GetRequest(snapshot, response.Id); - var tx = ExpressOracle.CreateResponseTx(snapshot, request, response, oracleNodes, neoSystem.Settings); + var tx = NodeUtility.CreateResponseTx(snapshot, request, response, oracleNodes, neoSystem.Settings); return tx == null ? null : Convert.ToBase64String(tx.ToArray()); } @@ -413,19 +410,14 @@ public JObject GetNep11Properties(JArray @params) [RpcMethod] public JObject? ExpressPersistContract(JObject @params) { - var state = RpcClient.ContractStateFromJson(@params[0]["state"]); - var storagePairs = new (byte[] Key, byte[] value)[0]; - - foreach (var pair in (JArray)@params[0]["storage"]) - { - storagePairs.Append(( - Convert.FromBase64String(pair["key"].AsString()), - Convert.FromBase64String(pair["value"].AsString()) - )); - } + var storagePairs = ((JArray)@params[0]["storage"]) + .Select(s => ( + s["key"].AsString(), + s["value"].AsString()) + ).ToArray(); - return ExpressOracle.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); + return NodeUtility.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); } static readonly IReadOnlySet nep11PropertyNames = new HashSet diff --git a/src/neoxp/Node/ExpressOracle.cs b/src/neoxp/Node/NodeUtility.cs similarity index 96% rename from src/neoxp/Node/ExpressOracle.cs rename to src/neoxp/Node/NodeUtility.cs index 09d11be6..b2a3b13d 100644 --- a/src/neoxp/Node/ExpressOracle.cs +++ b/src/neoxp/Node/NodeUtility.cs @@ -17,7 +17,7 @@ namespace NeoExpress.Node { - class ExpressOracle + class NodeUtility { public static void SignOracleResponseTransaction(ProtocolSettings settings, ExpressChain chain, Transaction tx, IReadOnlyList oracleNodes) { @@ -166,7 +166,7 @@ private static int GetNextAvailableId(DataCache snapshot) return value; } - public static bool PersistContract(SnapshotCache snapshot, ContractState state, (byte[] key, byte[] value)[] storagePairs) + public static int PersistContract(SnapshotCache snapshot, ContractState state, (string key, string value)[] storagePairs) { // Our local chain might already be using the contract id of the pulled contract, we need to check for this // to avoid having contracts with duplicate id's. This is important because the contract id is part of the @@ -188,12 +188,12 @@ public static bool PersistContract(SnapshotCache snapshot, ContractState state, foreach (var pair in storagePairs) { snapshot.Add( - new StorageKey { Id = state.Id, Key = pair.key}, - new StorageItem(pair.value) + new StorageKey { Id = state.Id, Key = Convert.FromBase64String(pair.key)}, + new StorageItem(Convert.FromBase64String(pair.value)) ); } snapshot.Commit(); - return true; + return state.Id; } } } diff --git a/src/neoxp/Node/OfflineNode.cs b/src/neoxp/Node/OfflineNode.cs index ea92312b..9342cbb9 100644 --- a/src/neoxp/Node/OfflineNode.cs +++ b/src/neoxp/Node/OfflineNode.cs @@ -179,9 +179,9 @@ public async Task SubmitOracleResponseAsync(OracleResponse response, IR using var snapshot = neoSystem.GetSnapshot(); var height = NativeContract.Ledger.CurrentIndex(snapshot) + 1; var request = NativeContract.Oracle.GetRequest(snapshot, response.Id); - var tx = ExpressOracle.CreateResponseTx(snapshot, request, response, oracleNodes, ProtocolSettings); + var tx = NodeUtility.CreateResponseTx(snapshot, request, response, oracleNodes, ProtocolSettings); if (tx == null) throw new Exception("Failed to create Oracle Response Tx"); - ExpressOracle.SignOracleResponseTransaction(ProtocolSettings, chain, tx, oracleNodes); + NodeUtility.SignOracleResponseTransaction(ProtocolSettings, chain, tx, oracleNodes); var blockHash = await SubmitTransactionAsync(tx); return tx.Hash; @@ -408,12 +408,12 @@ IReadOnlyList ListStorages(UInt160 scriptHash) public Task> ListStoragesAsync(UInt160 scriptHash) => MakeAsync(() => ListStorages(scriptHash)); - bool PersistContract(ContractState state, (byte[] key, byte[] value)[] storagePairs) + int PersistContract(ContractState state, (string key, string value)[] storagePairs) { - return ExpressOracle.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); + return NodeUtility.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); } - public Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs) + public Task PersistContractAsync(ContractState state, (string key, string value)[] storagePairs) => MakeAsync(() => PersistContract(state, storagePairs)); } diff --git a/src/neoxp/Node/OnlineNode.cs b/src/neoxp/Node/OnlineNode.cs index 4ec6df28..460ac770 100644 --- a/src/neoxp/Node/OnlineNode.cs +++ b/src/neoxp/Node/OnlineNode.cs @@ -196,7 +196,7 @@ public async Task SubmitOracleResponseAsync(OracleResponse response, IR { var jsonTx = await rpcClient.RpcSendAsync("expresscreateoracleresponsetx", response.ToJson()).ConfigureAwait(false); var tx = Convert.FromBase64String(jsonTx.AsString()).AsSerializable(); - ExpressOracle.SignOracleResponseTransaction(ProtocolSettings, chain, tx, oracleNodes); + NodeUtility.SignOracleResponseTransaction(ProtocolSettings, chain, tx, oracleNodes); return await rpcClient.SendRawTransactionAsync(tx).ConfigureAwait(false); } @@ -329,7 +329,7 @@ public async Task> ListStoragesAsync(UInt160 scrip return Array.Empty(); } - public async Task PersistContractAsync(ContractState state, (byte[] key, byte[] value)[] storagePairs) + public async Task PersistContractAsync(ContractState state, (string key, string value)[] storagePairs) { JObject o = new JObject(); o["state"] = state.ToJson(); @@ -338,14 +338,14 @@ public async Task PersistContractAsync(ContractState state, (byte[] key, b foreach (var pair in storagePairs) { JObject kv = new JObject(); - kv["key"] = Convert.ToBase64String(pair.key); - kv["value"] = Convert.ToBase64String(pair.value); + kv["key"] = pair.key; + kv["value"] = pair.value; storage.Add(kv); } o["storage"] = storage; - + var response = await rpcClient.RpcSendAsync("expresspersistcontract", o).ConfigureAwait(false); - return response.AsBoolean(); + return (int)response.AsNumber(); } } } From 061f2b40ff04716f05d2f50cec57021a8e311d38 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 22 Mar 2022 09:20:38 +0100 Subject: [PATCH 04/29] batch command --- .../BatchCommand.BatchFileCommands.cs | 16 ++++++- src/neoxp/Commands/BatchCommand.cs | 12 +++++ .../Commands/ContractCommand.Download.cs | 34 +++----------- src/neoxp/Node/NodeUtility.cs | 45 +++++++++++++++++++ 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs b/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs index b86cec6e..e373e367 100644 --- a/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs +++ b/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs @@ -29,7 +29,7 @@ internal class Create } [Command("contract")] - [Subcommand(typeof(Deploy), typeof(Invoke), typeof(Run))] + [Subcommand(typeof(Deploy), typeof(Download), typeof(Invoke), typeof(Run))] internal class Contract { [Command("deploy")] @@ -53,6 +53,20 @@ internal class Deploy [Option(Description = "Deploy contract regardless of name conflict")] internal bool Force { get; } } + + [Command("download")] + internal class Download + { + [Argument(0, Description = "Contract invocation hash")] + [Required] + internal string Contract { get; init; } = string.Empty; + + [Argument(1, Description = "URL of Neo JSON-RPC Node\nSpecify MainNet (default), TestNet or JSON-RPC URL")] + internal string RpcUri { get; } = string.Empty; + + [Argument(2, Description = "Block height to get contract state for")] + internal uint Height { get; } = 0; + } [Command("invoke")] internal class Invoke diff --git a/src/neoxp/Commands/BatchCommand.cs b/src/neoxp/Commands/BatchCommand.cs index e10df619..e1e19569 100644 --- a/src/neoxp/Commands/BatchCommand.cs +++ b/src/neoxp/Commands/BatchCommand.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using McMaster.Extensions.CommandLineUtils; +using NeoExpress.Node; namespace NeoExpress.Commands { @@ -113,6 +114,17 @@ await txExec.ContractDeployAsync( cmd.Model.Force).ConfigureAwait(false); break; } + case CommandLineApplication cmd: + { + var expressNode = chainManager.GetExpressNode(); + var result = await NodeUtility.ProcessDownloadParamsAsync( + cmd.Model.Contract, + cmd.Model.RpcUri, + cmd.Model.Height, + true).ConfigureAwait(false); + await expressNode.PersistContractAsync(result.contractState, result.storagePairs).ConfigureAwait(false); + break; + } case CommandLineApplication cmd: { var script = await txExec.LoadInvocationScriptAsync( diff --git a/src/neoxp/Commands/ContractCommand.Download.cs b/src/neoxp/Commands/ContractCommand.Download.cs index c9b8aa50..ff9a4875 100644 --- a/src/neoxp/Commands/ContractCommand.Download.cs +++ b/src/neoxp/Commands/ContractCommand.Download.cs @@ -1,12 +1,9 @@ using System; using System.ComponentModel.DataAnnotations; using System.IO; -using System.Net.Http; using System.Threading.Tasks; using McMaster.Extensions.CommandLineUtils; -using Neo; -using Neo.IO.Json; -using Neo.Network.RPC; +using NeoExpress.Node; namespace NeoExpress.Commands { @@ -31,36 +28,17 @@ public Download(ExpressChainManagerFactory chainManagerFactory) [Option(Description = "Path to neo-express data file")] internal string Input { get; init; } = string.Empty; + + [Option(Description = "Block height to get contract state for")] + internal uint Height { get; } = 0; internal async Task ExecuteAsync(TextWriter writer) { var (chainManager, _) = chainManagerFactory.LoadChain(Input); var expressNode = chainManager.GetExpressNode(); - if (!UInt160.TryParse(Contract, out var contractHash)) - { - throw new ArgumentException($"Invalid contract hash: \"{Contract}\""); - } - - if (!TransactionExecutor.TryParseRpcUri(RpcUri, out var uri)) - { - throw new ArgumentException($"Invalid RpcUri value \"{RpcUri}\""); - } - - using var rpcClient = new RpcClient(uri); - var stateAPI = new StateAPI(rpcClient); - - var stateHeight = await stateAPI.GetStateHeightAsync(); - if (stateHeight.localRootIndex is null) - { - throw new Exception("Null \"localRootIndex\" in state height response"); - } - - var stateRoot = await stateAPI.GetStateRootAsync(stateHeight.localRootIndex.Value); - var states = await rpcClient.ExpressFindStatesAsync(stateRoot.RootHash, contractHash, new byte[0]); - var contractState = await rpcClient.GetContractStateAsync(Contract).ConfigureAwait(false); - - await expressNode.PersistContractAsync(contractState, states.Results); + var result = await NodeUtility.ProcessDownloadParamsAsync(Contract, RpcUri, Height, false); + await expressNode.PersistContractAsync(result.contractState, result.storagePairs); } internal async Task OnExecuteAsync(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/Node/NodeUtility.cs b/src/neoxp/Node/NodeUtility.cs index b2a3b13d..ddb41c31 100644 --- a/src/neoxp/Node/NodeUtility.cs +++ b/src/neoxp/Node/NodeUtility.cs @@ -2,11 +2,13 @@ using System.Collections.Generic; using System.Linq; using System.Numerics; +using System.Threading.Tasks; using Neo; using Neo.BlockchainToolkit.Models; using Neo.Cryptography.ECC; using Neo.IO; using Neo.Network.P2P.Payloads; +using Neo.Network.RPC; using Neo.Persistence; using Neo.SmartContract; using Neo.SmartContract.Manifest; @@ -165,6 +167,49 @@ private static int GetNextAvailableId(DataCache snapshot) item.Add(1); return value; } + + public static async Task<(ContractState contractState, (string key, string value)[] storagePairs)> ProcessDownloadParamsAsync( + string contractHash, + string rpcUri, + uint stateHeight, + bool isBatchCommand) + { + if (!UInt160.TryParse(contractHash, out var contractHash_)) + { + throw new ArgumentException($"Invalid contract hash: \"{contractHash}\""); + } + + if (!TransactionExecutor.TryParseRpcUri(rpcUri, out var uri)) + { + throw new ArgumentException($"Invalid RpcUri value \"{rpcUri}\""); + } + + if (isBatchCommand && stateHeight == 0) + { + throw new ArgumentException("Height cannot be 0. Please specify a height > 0"); + } + + using var rpcClient = new RpcClient(uri); + var stateAPI = new StateAPI(rpcClient); + + uint height = stateHeight; + if (height == 0) + { + var stateHeight_ = await stateAPI.GetStateHeightAsync(); + if (stateHeight_.localRootIndex is null) + { + throw new Exception("Null \"localRootIndex\" in state height response"); + } + height = stateHeight_.localRootIndex.Value; + } + + var stateRoot = await stateAPI.GetStateRootAsync(height); + var states = await rpcClient.ExpressFindStatesAsync(stateRoot.RootHash, contractHash_, new byte[0]); + var contractState = await rpcClient.GetContractStateAsync(contractHash).ConfigureAwait(false); + + return (contractState, states.Results); + } + public static int PersistContract(SnapshotCache snapshot, ContractState state, (string key, string value)[] storagePairs) { From f4f2382ea547e34d0ae72c195e26431ab07c5155 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 22 Mar 2022 09:53:47 +0100 Subject: [PATCH 05/29] Resolve null warnings --- src/neoxp/Extensions/ExpressRpcFoundStates.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/neoxp/Extensions/ExpressRpcFoundStates.cs b/src/neoxp/Extensions/ExpressRpcFoundStates.cs index e15fd4ff..186a19d7 100644 --- a/src/neoxp/Extensions/ExpressRpcFoundStates.cs +++ b/src/neoxp/Extensions/ExpressRpcFoundStates.cs @@ -8,9 +8,9 @@ namespace NeoExpress public class ExpressRpcFoundStates { public bool Truncated; - public (string key, string value)[] Results; - public byte[]? FirstProof; - public byte[]? LastProof; + public (string key, string value)[] Results = new (string key, string value)[0]; + public (bool hasValue, byte[]? value) FirstProof; + public (bool hasValue, byte[]? value) LastProof; public static ExpressRpcFoundStates FromJson(JObject json) { @@ -28,7 +28,7 @@ public static ExpressRpcFoundStates FromJson(JObject json) }; } - static byte[] ProofFromJson(JObject json) - => json == null ? null : Convert.FromBase64String(json.AsString()); + private static (bool hasValue, byte[]? value) ProofFromJson(JObject json) + => json == null ? (false, null) : (true, Convert.FromBase64String(json.AsString())); } } \ No newline at end of file From 0c20e5d7e156fd5b52d5db4a17e01db7ab5df68d Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Mon, 28 Mar 2022 12:45:52 +0200 Subject: [PATCH 06/29] handle duplicate contracts --- .../BatchCommand.BatchFileCommands.cs | 6 ++ src/neoxp/Commands/BatchCommand.cs | 5 +- .../Commands/ContractCommand.Download.cs | 13 +++- src/neoxp/IExpressNode.cs | 3 +- src/neoxp/Node/Modules/ExpressRpcMethods.cs | 8 ++- src/neoxp/Node/NodeUtility.cs | 62 +++++++++++++------ src/neoxp/Node/OfflineNode.cs | 9 +-- src/neoxp/Node/OnlineNode.cs | 6 +- 8 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs b/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs index 8ba4acbf..7bd8cb60 100644 --- a/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs +++ b/src/neoxp/Commands/BatchCommand.BatchFileCommands.cs @@ -65,10 +65,16 @@ internal class Download internal string Contract { get; init; } = string.Empty; [Argument(1, Description = "URL of Neo JSON-RPC Node\nSpecify MainNet (default), TestNet or JSON-RPC URL")] + [Required] internal string RpcUri { get; } = string.Empty; [Argument(2, Description = "Block height to get contract state for")] + [Required] internal uint Height { get; } = 0; + + [Option(CommandOptionType.SingleOrNoValue, Description = "Replace contract and storage if it already exists (Default: All)")] + [AllowedValues(StringComparison.OrdinalIgnoreCase, "All", "ContractOnly", "StorageOnly")] + internal (bool hasValue, ContractCommand.ContractForce value) Force { get; init; } } [Command("invoke")] diff --git a/src/neoxp/Commands/BatchCommand.cs b/src/neoxp/Commands/BatchCommand.cs index ca332a90..319d0e76 100644 --- a/src/neoxp/Commands/BatchCommand.cs +++ b/src/neoxp/Commands/BatchCommand.cs @@ -131,7 +131,10 @@ await txExec.ContractDeployAsync( cmd.Model.RpcUri, cmd.Model.Height, true).ConfigureAwait(false); - await expressNode.PersistContractAsync(result.contractState, result.storagePairs).ConfigureAwait(false); + await expressNode.PersistContractAsync( + result.contractState, + result.storagePairs, + cmd.Model.Force.hasValue ? cmd.Model.Force.value : null).ConfigureAwait(false); break; } case CommandLineApplication cmd: diff --git a/src/neoxp/Commands/ContractCommand.Download.cs b/src/neoxp/Commands/ContractCommand.Download.cs index ff9a4875..521e4390 100644 --- a/src/neoxp/Commands/ContractCommand.Download.cs +++ b/src/neoxp/Commands/ContractCommand.Download.cs @@ -9,6 +9,13 @@ namespace NeoExpress.Commands { partial class ContractCommand { + internal enum ContractForce + { + All, + ContractOnly, + StorageOnly + } + [Command(Name = "download", Description = "Download contract with storage from remote chain into local chain")] internal class Download { @@ -32,13 +39,17 @@ public Download(ExpressChainManagerFactory chainManagerFactory) [Option(Description = "Block height to get contract state for")] internal uint Height { get; } = 0; + [Option(CommandOptionType.SingleOrNoValue, Description = "Replace contract and storage if it already exists (Default: All)")] + [AllowedValues(StringComparison.OrdinalIgnoreCase, "All", "ContractOnly", "StorageOnly")] + internal (bool hasValue, ContractForce value) Force { get; init; } + internal async Task ExecuteAsync(TextWriter writer) { var (chainManager, _) = chainManagerFactory.LoadChain(Input); var expressNode = chainManager.GetExpressNode(); var result = await NodeUtility.ProcessDownloadParamsAsync(Contract, RpcUri, Height, false); - await expressNode.PersistContractAsync(result.contractState, result.storagePairs); + await expressNode.PersistContractAsync(result.contractState, result.storagePairs, Force.hasValue ? Force.value : null); } internal async Task OnExecuteAsync(CommandLineApplication app, IConsole console) diff --git a/src/neoxp/IExpressNode.cs b/src/neoxp/IExpressNode.cs index bfaca4e7..9ce10b26 100644 --- a/src/neoxp/IExpressNode.cs +++ b/src/neoxp/IExpressNode.cs @@ -11,6 +11,7 @@ using Neo.SmartContract.Native; using Neo.VM; using Neo.Wallets; +using NeoExpress.Commands; using NeoExpress.Models; namespace NeoExpress @@ -41,7 +42,7 @@ enum CheckpointMode { Online, Offline } Task> ListStoragesAsync(UInt160 scriptHash); Task> ListTokenContractsAsync(); - Task PersistContractAsync(ContractState state, (string key, string value)[] storagePairs); + Task PersistContractAsync(ContractState state, (string key, string value)[] storagePairs, ContractCommand.ContractForce? force); IAsyncEnumerable<(uint blockIndex, NotificationRecord notification)> EnumerateNotificationsAsync(IReadOnlySet? contractFilter, IReadOnlySet? eventFilter); } } diff --git a/src/neoxp/Node/Modules/ExpressRpcMethods.cs b/src/neoxp/Node/Modules/ExpressRpcMethods.cs index 8c7556b8..5ad855d0 100644 --- a/src/neoxp/Node/Modules/ExpressRpcMethods.cs +++ b/src/neoxp/Node/Modules/ExpressRpcMethods.cs @@ -14,6 +14,7 @@ using Neo.SmartContract.Native; using Neo.VM; using Neo.Wallets; +using NeoExpress.Commands; using NeoExpress.Models; using ByteString = Neo.VM.Types.ByteString; using RpcException = Neo.Plugins.RpcException; @@ -455,7 +456,7 @@ public JObject GetNep11Properties(JArray @params) } [RpcMethod] - public JObject? ExpressPersistContract(JObject @params) + public JObject ExpressPersistContract(JObject @params) { var state = RpcClient.ContractStateFromJson(@params[0]["state"]); var storagePairs = ((JArray)@params[0]["storage"]) @@ -463,8 +464,11 @@ public JObject GetNep11Properties(JArray @params) s["key"].AsString(), s["value"].AsString()) ).ToArray(); + ContractCommand.ContractForce? force = @params[0]["force"] is null ? + null : + Enum.Parse(@params[0]["force"].AsString()); - return NodeUtility.PersistContract(neoSystem.GetSnapshot(), state, storagePairs); + return NodeUtility.PersistContract(neoSystem.GetSnapshot(), state, storagePairs, force); } static readonly IReadOnlySet nep11PropertyNames = new HashSet diff --git a/src/neoxp/Node/NodeUtility.cs b/src/neoxp/Node/NodeUtility.cs index ddb41c31..b9c360c5 100644 --- a/src/neoxp/Node/NodeUtility.cs +++ b/src/neoxp/Node/NodeUtility.cs @@ -15,6 +15,7 @@ using Neo.SmartContract.Native; using Neo.VM; using Neo.Wallets; +using NeoExpress.Commands; using NeoExpress.Models; namespace NeoExpress.Node @@ -211,31 +212,54 @@ private static int GetNextAvailableId(DataCache snapshot) } - public static int PersistContract(SnapshotCache snapshot, ContractState state, (string key, string value)[] storagePairs) + public static int PersistContract(SnapshotCache snapshot, ContractState state, (string key, string value)[] storagePairs, ContractCommand.ContractForce? force) { - // Our local chain might already be using the contract id of the pulled contract, we need to check for this - // to avoid having contracts with duplicate id's. This is important because the contract id is part of the - // StorageContext used with Storage syscalls and else we'll potentially override storage keys or iterate - // over keys that shouldn't exist for one of the contracts. - if (state.Id <= LastUsedContractId(snapshot)) + var localContract = NativeContract.ContractManagement.GetContract(snapshot, state.Hash); + + if (localContract is null) { - state.Id = GetNextAvailableId(snapshot); + // Our local chain might already be using the contract id of the pulled contract, we need to check for this + // to avoid having contracts with duplicate id's. This is important because the contract id is part of the + // StorageContext used with Storage syscalls and else we'll potentially override storage keys or iterate + // over keys that shouldn't exist for one of the contracts. + if (state.Id < LastUsedContractId(snapshot)) + { + state.Id = GetNextAvailableId(snapshot); + } + else + { + // Update available id such that a regular contract deploy will use the right next id; + SetLastUsedContractId(snapshot, state.Id); + } + + StorageKey key = new KeyBuilder(NativeContract.ContractManagement.Id, Prefix_Contract).Add(state.Hash); + snapshot.Add(key, new StorageItem(state)); } - else + else { - // Update available id such that a regular contract deploy will use the right next id; - SetLastUsedContractId(snapshot, state.Id); + if (force is null) + { + throw new Exception("Contract already exists locally. Use --force: